From 5d4af81c3e6b5d9827adb2a87593cc04722fb0b6 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 6 Sep 2026 09:03:39 +0200 Subject: [PATCH 01/13] Dynamic Change Requests (Maker/checker for dynamic code, approved_hash) - Updates to signal channels with added gRPC service. --- obp-api/src/main/protobuf/signal.proto | 7 + .../resources/props/sample.props.template | 26 + .../props/test.default.props.template | 2 + .../main/scala/bootstrap/liftweb/Boot.scala | 5 + .../scala/code/abacrule/AbacRuleEngine.scala | 4 + .../scala/code/abacrule/AbacRuleTrait.scala | 3 + .../SwaggerDefinitionsJSON.scala | 8 +- .../scala/code/api/cache/RedisMessaging.scala | 125 ++++- .../DynamicResourceDocsEndpointGroup.scala | 6 +- .../main/scala/code/api/util/ApiRole.scala | 6 + .../src/main/scala/code/api/util/ApiTag.scala | 1 + .../scala/code/api/util/ErrorMessages.scala | 13 + .../main/scala/code/api/util/Glossary.scala | 6 +- .../code/api/util/http4s/Http4sSupport.scala | 22 + .../scala/code/api/v4_0_0/Http4s400.scala | 140 +++-- .../scala/code/api/v6_0_0/Http4s600.scala | 153 ++--- .../code/api/v6_0_0/JSONFactory6.0.0.scala | 15 +- .../scala/code/api/v7_0_0/Http4s700.scala | 415 ++++++++++++++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 91 ++- .../DoobieBusinessStatusQueries.scala | 18 + .../bankconnectors/DynamicConnector.scala | 4 +- .../bankconnectors/InternalConnector.scala | 4 +- .../connectormethod/ConnectorMethod.scala | 8 + .../dynamicMessageDoc/DynamicMessageDoc.scala | 8 + .../DynamicResourceDoc.scala | 8 + .../DynamicChangeRequest.scala | 129 +++++ .../DynamicChangeRequestTrait.scala | 77 +++ .../dynamicchangerequest/MakerChecker.scala | 521 ++++++++++++++++++ .../scala/code/obp/grpc/ObpGrpcServer.scala | 12 +- .../signal/SignalChannelsServiceImpl.scala | 157 ++++++ .../obp/grpc/signal/api/FetchRequest.scala | 146 +++++ .../obp/grpc/signal/api/FetchResponse.scala | 190 +++++++ .../grpc/signal/api/ListChannelsRequest.scala | 86 +++ .../signal/api/ListChannelsResponse.scala | 105 ++++ .../obp/grpc/signal/api/PublishRequest.scala | 146 +++++ .../obp/grpc/signal/api/PublishResponse.scala | 177 ++++++ .../grpc/signal/api/SignalChannelInfo.scala | 129 +++++ .../api/SignalChannelsServiceGrpc.scala | 163 ++++++ .../obp/grpc/signal/api/SignalMessage.scala | 245 ++++++++ .../obp/grpc/signal/api/SignalProto.scala | 159 ++++++ .../grpc/signal/api/SubscribeRequest.scala | 95 ++++ .../scala/code/signal/SignalChannels.scala | 95 ++++ .../scala/code/signal/SignalEventBus.scala | 111 ++++ .../main/scala/code/users/UserReference.scala | 5 + .../code/api/v6_0_0/SignalChannelTest.scala | 48 +- .../api/v7_0_0/DynamicChangeRequestTest.scala | 343 ++++++++++++ .../obp/grpc/SignalChannelsGrpcTest.scala | 232 ++++++++ .../commons/model/enums/Enumerations.scala | 20 + 48 files changed, 4345 insertions(+), 144 deletions(-) create mode 100644 obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequest.scala create mode 100644 obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequestTrait.scala create mode 100644 obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/FetchRequest.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsRequest.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsResponse.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/PublishRequest.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/PublishResponse.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelInfo.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/SignalMessage.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala create mode 100644 obp-api/src/main/scala/code/obp/grpc/signal/api/SubscribeRequest.scala create mode 100644 obp-api/src/main/scala/code/signal/SignalChannels.scala create mode 100644 obp-api/src/main/scala/code/signal/SignalEventBus.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala create mode 100644 obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala diff --git a/obp-api/src/main/protobuf/signal.proto b/obp-api/src/main/protobuf/signal.proto index 81e01427cb..a8bf0a8b2b 100644 --- a/obp-api/src/main/protobuf/signal.proto +++ b/obp-api/src/main/protobuf/signal.proto @@ -15,6 +15,7 @@ message SignalMessage { google.protobuf.Timestamp timestamp = 6; string message_type = 7; string payload_json = 8; // JSON-encoded payload + int64 sequence = 9; // per-channel monotonic; poll with FetchRequest.after_sequence } // Mirrors SignalChannelInfoJsonV600 @@ -38,6 +39,7 @@ message PublishResponse { string channel_name = 2; google.protobuf.Timestamp timestamp = 3; int64 channel_message_count = 4; + int64 sequence = 5; } // --- Fetch: 1:1 with GET /signal/channels/{name}/messages --- @@ -48,6 +50,9 @@ message FetchRequest { string channel_name = 1; int32 offset = 2; int32 limit = 3; + // > 0: cursor mode, return messages with sequence > after_sequence and ignore offset. + // Prefer this for polling: offset paging drifts once the channel is trimmed. + int64 after_sequence = 4; } message FetchResponse { @@ -55,6 +60,8 @@ message FetchResponse { repeated SignalMessage messages = 2; int64 total_count = 3; bool has_more = 4; + int64 latest_sequence = 5; // newest message in the channel, 0 when empty + int64 next_after_sequence = 6; // pass back as after_sequence to continue (advances past hidden private messages too) } // --- ListChannels: 1:1 with GET /signal/channels --- diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 374b071ca9..277b40800c 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1739,6 +1739,32 @@ dynamic_code_compile_validate_dependencies=[\ PractiseEndpoint.getClass.getTypeName + "*" -> "*"\ ] +# --- Dynamic code requires approval (maker/checker, see MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md) --- +# dynamic_code_requires_approval=true enforces two things for the target types listed below: +# 1. Writes are queued, not applied. Create/update/delete via the v4.0.0/v6.0.0 endpoints still check the +# caller's role, validate and compile the body, then store a DynamicChangeRequest and answer 202 Accepted. +# A DIFFERENT user holding CanApproveDynamicChangeRequest applies it by quoting the payload's SHA-256 at +# /obp/v7.0.0/management/dynamic-change-requests/ID/approval. +# 2. The runtime executes only approved code. A row is served/compiled only when it is active and its body +# hash equals the hash a checker approved. Rows edited or inserted directly in the database are not run. +# The first boot with this true seeds the approved hash of every pre-existing row from its current body, +# once per database (logged in MigrationScriptLog as seedDynamicCodeApprovedHashes). After that, the only way +# a row becomes executable is a checker's approval (or an ACTIVATE change request for a row that has none). +# Approval is system level: dynamic code runs in the shared JVM, so a bank-level artefact is still approved by +# a system-level checker. Deactivation is a direct action by a single approver (four eyes to enable, one pair +# to disable) and works whether or not this prop is set. +# Defaults to false: sandboxes and local development keep today's behaviour, nothing is queued or gated. +dynamic_code_requires_approval=false +# Which target types the above applies to. Phase 1 supports the four code families. +dynamic_code_approval_target_types=DYNAMIC_RESOURCE_DOC,DYNAMIC_MESSAGE_DOC,CONNECTOR_METHOD,ABAC_RULE +# Deleting does not expand capability but does break consumers; set false to let makers delete directly. +dynamic_code_delete_requires_approval=true +# INITIATED requests older than this are marked EXPIRED when next read. 0 disables expiry. +dynamic_code_approval_request_ttl_hours=168 +# Connector methods and dynamic message docs are looked up per call, so their active/approved check is +# memoised for this long. An approval or deactivation takes up to this long to reach those two families. +dynamic_code_approval_guard_cache_ttl_seconds=10 + ################################################### ## "Optional" / "Placeholder" JSON field behaviour # Sometimes our connectors or data imports might populate fields with default, null or placeholder values such as empty strings, default dates and empty lists diff --git a/obp-api/src/main/resources/props/test.default.props.template b/obp-api/src/main/resources/props/test.default.props.template index 21f8856d41..4a9812f382 100644 --- a/obp-api/src/main/resources/props/test.default.props.template +++ b/obp-api/src/main/resources/props/test.default.props.template @@ -152,6 +152,8 @@ hikari.maximumPoolSize=20 # ConnectorMethodTest, AbacRuleTests, DynamicResourceDocTest, DynamicMessageDocTest and # DynamicCodeKillSwitchTest's ON scenarios can compile/execute dynamic code locally. allow_user_generated_scala_code=true +# Maker/checker for dynamic code is off by default; DynamicChangeRequestTest turns it on per scenario. +dynamic_code_requires_approval=false # Permissions granted to runtime-compiled dynamic-endpoint code inside the security sandbox. # Mirrors default.props / production.default.props. Required so dynamic resource-doc bodies can do diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index c339fb6087..a9b96b4966 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -302,6 +302,10 @@ class Boot extends MdcLoggable { // Please note that migration scripts are executed after Lift Mapper Schemifier Migration.database.executeScripts(startedBeforeSchemifier = false) + // Maker/checker for dynamic code: when first enabled, pre-existing code rows get their current + // body hash recorded as approved so enabling the feature does not silently disable them. + code.dynamicchangerequest.MakerChecker.seedApprovedHashesIfEnabled() + // Idempotent seed of country-qualified routing schemes (TZ.MSISDN, bill, utility, etc.). // Toggle off via routing_schemes.seed_defaults_at_boot=false in environments that don't want defaults. code.routingscheme.RoutingSchemeSeed.runIfEnabled() @@ -1081,6 +1085,7 @@ object ToSchemify extends MdcLoggable { BulkPayment, BulkBatchReference, AccountAccessRequest, + code.dynamicchangerequest.DynamicChangeRequest, code.chat.ChatRoom, code.chat.Participant, code.chat.ChatMessage, diff --git a/obp-api/src/main/scala/code/abacrule/AbacRuleEngine.scala b/obp-api/src/main/scala/code/abacrule/AbacRuleEngine.scala index bbd9b72906..88bdb96f4c 100644 --- a/obp-api/src/main/scala/code/abacrule/AbacRuleEngine.scala +++ b/obp-api/src/main/scala/code/abacrule/AbacRuleEngine.scala @@ -44,6 +44,10 @@ object AbacRuleEngine { * @return Box containing the compiled function or error */ def compileRule(ruleId: String, ruleCode: String): Box[AbacRuleFunction] = { + // Maker/checker execution guard: on a managed instance a rule whose current code hash differs + // from the checker-approved hash is never compiled or run (see MakerChecker.isApprovedAbacRule). + if (!code.dynamicchangerequest.MakerChecker.isApprovedAbacRule(ruleId)) + return Failure(ErrorMessages.DynamicArtefactNotApproved) compiledRulesCache.get(ruleId) match { case Some(cachedFunction) => cachedFunction case None => diff --git a/obp-api/src/main/scala/code/abacrule/AbacRuleTrait.scala b/obp-api/src/main/scala/code/abacrule/AbacRuleTrait.scala index 8a0e995f99..1609ec032a 100644 --- a/obp-api/src/main/scala/code/abacrule/AbacRuleTrait.scala +++ b/obp-api/src/main/scala/code/abacrule/AbacRuleTrait.scala @@ -34,6 +34,9 @@ class AbacRule extends AbacRuleTrait with LongKeyedMapper[AbacRule] with IdPK wi object Policy extends MappedText(this) object CreatedByUserId extends MappedString(this, 255) object UpdatedByUserId extends MappedString(this, 255) + // Maker/checker: SHA-256 of the RuleCode that a checker approved. When maker/checker is enabled + // for ABAC_RULE the engine refuses to compile a rule whose current code hash differs from this. + object ApprovedHash extends MappedString(this, 64) override def abacRuleId: String = AbacRuleId.get override def ruleName: String = RuleName.get diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 68e1d75eab..40b823ccbb 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -6463,6 +6463,7 @@ object SwaggerDefinitionsJSON { lazy val signalMessageJsonV600 = SignalMessageJsonV600( message_id = "d8839721-2e41-4c60-9bba-42c5a7164027", + sequence = 1771583400123456L, channel_name = "discovery", sender_consumer_id = "7uy8a7e4-6d02-40e3-a129-0b2bf89de8uh", sender_user_id = "9ca9a7e4-6d02-40e3-a129-0b2bf89de9b1", @@ -6476,14 +6477,17 @@ object SwaggerDefinitionsJSON { channel_name = "discovery", messages = List(signalMessageJsonV600), total_count = 1, - has_more = false + has_more = false, + latest_sequence = 1771583400123456L, + next_after_sequence = 1771583400123456L ) lazy val signalMessagePublishedJsonV600 = SignalMessagePublishedJsonV600( message_id = "d8839721-2e41-4c60-9bba-42c5a7164027", channel_name = "discovery", timestamp = "2026-02-20T10:30:00Z", - channel_message_count = 1 + channel_message_count = 1, + sequence = 1771583400123456L ) lazy val signalChannelInfoJsonV600 = SignalChannelInfoJsonV600( diff --git a/obp-api/src/main/scala/code/api/cache/RedisMessaging.scala b/obp-api/src/main/scala/code/api/cache/RedisMessaging.scala index 018d1a4276..0f3d46c559 100644 --- a/obp-api/src/main/scala/code/api/cache/RedisMessaging.scala +++ b/obp-api/src/main/scala/code/api/cache/RedisMessaging.scala @@ -16,6 +16,47 @@ object RedisMessaging extends MdcLoggable { private def channelKey(channelName: String): String = s"${keyPrefix}${channelName}" + // Deliberately NOT under keyPrefix: listChannels() globs keyPrefix* and must not see these. + private def sequenceKey(channelName: String): String = + s"${Constant.getGlobalCacheNamespacePrefix}signal_seq_${channelName}" + + private val SequencePrefix = """^\{"sequence":(\d+),""".r + + /** + * The sequence stamped on a stored envelope. 0 for envelopes stored before sequences + * existed, which sort as older than everything and are skipped by cursor reads. + */ + def sequenceOf(storedJson: String): Long = + Option(storedJson).flatMap(SequencePrefix.findFirstMatchIn(_)).map(_.group(1).toLong).getOrElse(0L) + + /** + * Publish as one atomic Lua script so the sequence, the push, the trim, the TTL refresh and the + * pub/sub fan-out cannot interleave with another publisher: + * + * - sequence = Redis server time in microseconds, forced strictly greater than the channel's + * previous sequence. Time-based rather than a counter so a cursor stays valid across a channel + * expiring and being recreated (a counter would restart at 1 and strand old cursors). + * - the sequence is spliced into the JSON as its first field; the envelope itself never carries + * one, so there is exactly one "sequence" key in what is stored and fanned out. + * - string.format('%d') everywhere a number becomes text: Lua 5.1 prints large numbers as + * 1.7e+15 otherwise. + */ + private val publishScript: String = + """local listKey, seqKey = KEYS[1], KEYS[2] + |local msg, maxMessages, ttl, pubsubChannel = ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3]), ARGV[4] + |local t = redis.call('TIME') + |local seq = tonumber(t[1]) * 1000000 + tonumber(t[2]) + |local last = tonumber(redis.call('GET', seqKey) or '0') + |if seq <= last then seq = last + 1 end + |local seqStr = string.format('%d', seq) + |redis.call('SET', seqKey, seqStr, 'EX', ttl) + |local stamped = '{"sequence":' .. seqStr .. ',' .. string.sub(msg, 2) + |redis.call('RPUSH', listKey, stamped) + |redis.call('LTRIM', listKey, -maxMessages, -1) + |redis.call('EXPIRE', listKey, ttl) + |redis.call('PUBLISH', pubsubChannel, stamped) + |return {seqStr, redis.call('LLEN', listKey)}""".stripMargin + def validateChannelName(name: String): Boolean = { name.nonEmpty && name.length <= 128 && @@ -23,27 +64,26 @@ object RedisMessaging extends MdcLoggable { } /** - * Publish a message to a channel. - * Uses RPUSH so messages are ordered oldest-first (index 0 = oldest). - * LTRIM caps the list at max messages. EXPIRE refreshes the TTL. + * Publish a message to a channel: stamp a sequence, RPUSH (oldest first), LTRIM to the newest + * `maxMessages`, refresh the TTL and PUBLISH for live gRPC subscribers — atomically, see + * publishScript. `messageJson` must be a JSON object without a "sequence" field. * - * @return the length of the list after push + * @param maxMessages overridable so a test can prove cursor reads survive trimming + * @return (sequence stamped on the message, length of the list after push) */ - def publishMessage(channelName: String, messageJson: String): Long = { + def publishMessage(channelName: String, messageJson: String, maxMessages: Int = channelMaxMessages): (Long, Long) = { + require(messageJson.startsWith("{"), "signal envelope must be a JSON object") var jedisConnection: Option[Jedis] = None try { jedisConnection = Some(Redis.jedisPool.getResource()) val jedis = jedisConnection.get - val key = channelKey(channelName) - - val length = jedis.rpush(key, messageJson) - // Cap the list: keep only the last N messages - jedis.ltrim(key, -channelMaxMessages.toLong, -1) - // Refresh TTL on every publish - jedis.expire(key, channelTtlSeconds) - // Pub/sub notification for live gRPC subscribers — fire-and-forget, no persistence - jedis.publish(s"obp_signal:$channelName", messageJson) - length + val result = jedis.eval( + publishScript, + java.util.Arrays.asList(channelKey(channelName), sequenceKey(channelName)), + java.util.Arrays.asList(messageJson, maxMessages.toString, channelTtlSeconds.toString, + code.signal.SignalEventBus.redisChannel(channelName)) + ).asInstanceOf[java.util.List[AnyRef]] + (result.get(0).toString.toLong, result.get(1).asInstanceOf[java.lang.Long].longValue()) } catch { case e: Throwable => logger.error(s"RedisMessaging.publishMessage error for channel $channelName: ${e.getMessage}") @@ -78,6 +118,57 @@ object RedisMessaging extends MdcLoggable { } } + /** Sequence of the newest message in a channel, 0 when the channel is empty or missing. */ + def latestSequence(channelName: String): Long = { + var jedisConnection: Option[Jedis] = None + try { + jedisConnection = Some(Redis.jedisPool.getResource()) + sequenceOf(jedisConnection.get.lindex(channelKey(channelName), -1)) + } catch { + case e: Throwable => + logger.error(s"RedisMessaging.latestSequence error for channel $channelName: ${e.getMessage}") + throw new RuntimeException(e) + } finally { + jedisConnection.foreach(_.close()) + } + } + + /** + * Cursor read: up to `limit` messages whose sequence is greater than `afterSequence`, oldest + * first. Unlike offset paging this is unaffected by LTRIM moving list indexes, because the + * position is found by binary search over the (strictly increasing) stamped sequences — + * at most log2(channelMaxMessages) LINDEX calls. + * + * @return (messages, total count in channel, latest sequence in channel) + */ + def fetchMessagesAfter(channelName: String, afterSequence: Long, limit: Int): (List[String], Long, Long) = { + var jedisConnection: Option[Jedis] = None + try { + jedisConnection = Some(Redis.jedisPool.getResource()) + val jedis = jedisConnection.get + val key = channelKey(channelName) + val total = jedis.llen(key) + if (total == 0L) (Nil, 0L, 0L) + else { + val latest = sequenceOf(jedis.lindex(key, -1)) + var lo = 0L + var hi = total + while (lo < hi) { + val mid = (lo + hi) / 2 + if (sequenceOf(jedis.lindex(key, mid)) <= afterSequence) lo = mid + 1 else hi = mid + } + val messages = if (lo >= total) Nil else jedis.lrange(key, lo, lo + limit - 1).asScala.toList + (messages, total, latest) + } + } catch { + case e: Throwable => + logger.error(s"RedisMessaging.fetchMessagesAfter error for channel $channelName: ${e.getMessage}") + throw new RuntimeException(e) + } finally { + jedisConnection.foreach(_.close()) + } + } + /** * List all active channel names by scanning for the key prefix. * @@ -110,8 +201,8 @@ object RedisMessaging extends MdcLoggable { try { jedisConnection = Some(Redis.jedisPool.getResource()) val jedis = jedisConnection.get - val key = channelKey(channelName) - jedis.del(key) > 0 + jedis.del(sequenceKey(channelName)) + jedis.del(channelKey(channelName)) > 0 } catch { case e: Throwable => logger.error(s"RedisMessaging.deleteChannel error for channel $channelName: ${e.getMessage}") diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 8e6350494b..b032422108 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -17,7 +17,11 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel // (request.json / Box[JsonResponse] / Full(errorJsonResponse(...))) will fail to compile under // the native http4s template. Skip (and log) such a row so one bad endpoint does not crash the // whole group / server boot. Re-author the body against the new native contract (see PractiseEndpoint). - DynamicResourceDocProvider.provider.vend.getAll(None).flatMap { dynamicDoc => + // Maker/checker execution guard: only active rows whose body hash a checker approved are served + // (the hash check applies only when dynamic_code_requires_approval covers DYNAMIC_RESOURCE_DOC). + DynamicResourceDocProvider.provider.vend.getAll(None) + .filter(_.dynamicResourceDocId.exists(code.dynamicchangerequest.MakerChecker.isExecutableDynamicResourceDoc)) + .flatMap { dynamicDoc => try { Some(toResourceDoc(dynamicDoc)) } catch { diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index 4a09586349..9b840a726b 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -1465,6 +1465,12 @@ object ApiRole extends MdcLoggable{ case class CanUpdateAccountAccessRequestAtAnyBank(requiresBankId: Boolean = false) extends ApiRole lazy val canUpdateAccountAccessRequestAtAnyBank = CanUpdateAccountAccessRequestAtAnyBank() + + // Maker/checker for dynamic code: approval is system level only (dynamic code runs in the shared JVM) + case class CanApproveDynamicChangeRequest(requiresBankId: Boolean = false) extends ApiRole + lazy val canApproveDynamicChangeRequest = CanApproveDynamicChangeRequest() + case class CanGetDynamicChangeRequests(requiresBankId: Boolean = false) extends ApiRole + lazy val canGetDynamicChangeRequests = CanGetDynamicChangeRequests() case class CanUpdateAccountAccessRequestAtOneBank(requiresBankId: Boolean = true) extends ApiRole lazy val canUpdateAccountAccessRequestAtOneBank = CanUpdateAccountAccessRequestAtOneBank() diff --git a/obp-api/src/main/scala/code/api/util/ApiTag.scala b/obp-api/src/main/scala/code/api/util/ApiTag.scala index a5da4c3017..059cbb4aa2 100644 --- a/obp-api/src/main/scala/code/api/util/ApiTag.scala +++ b/obp-api/src/main/scala/code/api/util/ApiTag.scala @@ -26,6 +26,7 @@ object ApiTag { val apiTagAccountAttribute = ResourceDocTag("Account-Attribute") val apiTagAccountAccess = ResourceDocTag("Account-Access") val apiTagAccountAccessRequest = ResourceDocTag("Account-Access-Request") + val apiTagDynamicChangeRequest = ResourceDocTag("Dynamic-Change-Request") val apiTagDirectDebit = ResourceDocTag("Direct-Debit") val apiTagStandingOrder = ResourceDocTag("Standing-Order") val apiTagAccountMetadata = ResourceDocTag("Account-Metadata") diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index dc8d1aea47..44c489c6ab 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -736,6 +736,19 @@ object ErrorMessages { val SmtpRecipientRejected = "OBP-30344: SMTP server rejected the recipient or message. The From address may be unauthorised, the recipient may be invalid, or the message may have failed policy/anti-spam checks." val SmtpProtocolError = "OBP-30345: SMTP protocol error from the mail server." + // Maker/checker for dynamic code and configuration (DynamicChangeRequest) + val DynamicChangeRequestNotFound = "OBP-30346: Dynamic change request not found. Please specify a valid CHANGE_REQUEST_ID." + val DynamicChangeRequestNotInitiated = "OBP-30347: Dynamic change request is not in INITIATED status. It has already been approved, rejected, withdrawn or has expired." + val DynamicChangeRequestHashMismatch = "OBP-30348: The payload_hash in the approval does not match the stored hash of the change request. Re-read the request and approve exactly the content shown." + val DynamicChangeRequestStale = "OBP-30349: The target of this change request has changed since it was submitted. Withdraw it and submit a new request against the current version." + val DynamicChangeRequestTargetTypeNotManaged = "OBP-30350: This target type is not managed by maker/checker on this instance (see dynamic_code_approval_target_types)." + val DynamicChangeRequestApprovalRequired = "OBP-30351: Maker/checker is enabled for this target type. The change has been queued as a dynamic change request and must be approved by a second user before it takes effect." + val DynamicChangeRequestTargetNotFound = "OBP-30352: The target of the dynamic change request does not exist." + val DynamicChangeRequestNotRequestor = "OBP-30353: Only the requestor of a dynamic change request can withdraw it." + val DynamicChangeRequestApplyFailed = "OBP-30354: The dynamic change request was approved but could not be applied. Its status is now FAILED; see checker_comment for the reason." + val DynamicArtefactInactive = "OBP-30355: This dynamic artefact is deactivated and will not be executed." + val DynamicArtefactNotApproved = "OBP-30356: This dynamic artefact's current code has not been approved by a checker and will not be executed." + // Branch related messages val BranchesNotFoundLicense = "OBP-32001: No branches available. License may not be set." val BranchesNotFound = "OBP-32002: No branches available." diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index b73124b256..df4f28b492 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -6044,6 +6044,7 @@ object Glossary extends MdcLoggable { |- Channels are auto-created on first publish; no registration step. |- On this instance a channel expires ${code.api.cache.RedisMessaging.channelTtlSeconds} seconds after its last publish, and holds at most ${code.api.cache.RedisMessaging.channelMaxMessages} messages (oldest are trimmed). |- Channel names are 1 to 128 characters from letters, digits, dot, underscore and hyphen. + |- Every message carries a per-channel monotonic **sequence** (Redis server time in microseconds, forced strictly increasing) stamped atomically when it is stored. Poll with `after_sequence=` and continue from the response's `next_after_sequence`. Do not poll by offset: trimming moves list positions, so an offset-tracking poller silently skips messages once the channel is full. Sequences are time-based rather than a counter so a cursor stays valid across a channel expiring and being recreated. | |## Constraints on published messages |All publishing requires authentication. Beyond that, three server-side checks protect the platform — the envelope, not the meaning, of what agents say: @@ -6066,7 +6067,10 @@ object Glossary extends MdcLoggable { |Signal channels are readable and writable by any authenticated consumer on the instance. If your agent feeds received payloads to an LLM, treat them as **untrusted data, never as instructions** — the character checks above stop display-layer trickery, but no server-side check can stop a payload from *saying* something misleading. Prompt-injection defence belongs in the consuming agent. | |## Endpoints - |See the API Explorer tags **Signal** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal/channels/...`. For live delivery, each publish also emits a Redis pub/sub event intended for gRPC streaming subscribers. + |See the API Explorer tags **Signal** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal/channels/...`. + | + |## gRPC + |The same operations are served over gRPC by `SignalChannelsService` (package `code.obp.grpc.signal.g1`, contract in `signal.proto`) when the gRPC server is enabled (`grpc.server.enabled`): **Publish**, **Fetch** and **ListChannels** are 1:1 with the REST endpoints and share their storage, and **Subscribe** is a server-side stream of new messages on one channel. Subscribe is live only — no catch-up, no replay — and applies the same privacy filter as Fetch. Each publish, REST or gRPC, is pushed to subscribers through Redis pub/sub. Authenticate with the same `Authorization` value the REST endpoints take, sent as gRPC metadata. | """) diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index 5aafa0b21a..3c61134aff 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -484,6 +484,28 @@ object Http4sRequestAttributes { } } + /** + * Execute business logic requiring validated User that returns a (result, statusCode) pair. + * A 204 renders with no body; any other status renders the result as JSON. Used by write + * endpoints that answer 201/200/204 when applied directly and 202 Accepted when maker/checker + * queues the change as a DynamicChangeRequest instead. + */ + def withUserAndStatus[A](req: Request[IO])(f: (User, CallContext) => Future[(A, Int)])(implicit formats: Formats): IO[Response[IO]] = { + implicit val cc: CallContext = req.callContext + val io = for { + user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) + result <- RequestScopeConnection.fromFuture(f(user, cc)) + } yield result + io.attempt.flatMap { + case Right((_, 204)) => NoContent().flatTap(recordMetric("", _)) + case Right((result, code)) => + val jsonString = prettyRender(Extraction.decompose(result)) + val status = Status.fromInt(code).getOrElse(Status.Ok) + IO.pure(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType)).flatTap(recordMetric(result, _)) + case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) + } + } + /** * Execute DELETE business logic (no auth required). * Returns 204 No Content on success, converts errors via ErrorResponseConverter. diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 4c94d61999..9c36f2a3ea 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -57,6 +57,10 @@ import code.model.dataAccess.BankAccountCreation import code.connectormethod.{JsonConnectorMethod, JsonConnectorMethodMethodBody} import code.dynamicMessageDoc.JsonDynamicMessageDoc import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.dynamicchangerequest.MakerChecker +import code.api.v7_0_0.JSONFactory700.createDynamicChangeRequestJsonV700 +import com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.{CONNECTOR_METHOD, DYNAMIC_MESSAGE_DOC, DYNAMIC_RESOURCE_DOC} +import com.openbankproject.commons.model.enums.{DynamicChangeRequestOperation => ChangeOp} import code.userlocks.UserLocksProvider import code.util.JsonSchemaUtil import code.validation.JsonValidation @@ -9215,7 +9219,7 @@ object Http4s400 { lazy val createConnectorMethod: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "connector-methods" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { val cc = req.callContext val rawBody = cc.httpBody.getOrElse("") for { @@ -9239,14 +9243,17 @@ object Http4s400 { else "" _ <- code.util.Helper.booleanToFuture(errorMsg, cc = callContext) { connectorMethod.isDefined } _ = Validation.validateDependency(connectorMethod.head) - (created, _) <- NewStyle.function.createJsonConnectorMethod(jsonConnectorMethod, callContext) - } yield created + result <- interceptOrApply(CONNECTOR_METHOD, ChangeOp.CREATE, None, 201, cc) { + NewStyle.function.createJsonConnectorMethod(jsonConnectorMethod, callContext).map(_._1) + } + } yield result } } lazy val updateConnectorMethod: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "connector-methods" / connectorMethodId => - EndpointHelpers.executeAndRespond(req) { cc => + EndpointHelpers.executeFutureWithStatus(req) { + val cc = req.callContext val rawBody = cc.httpBody.getOrElse("") for { _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled } @@ -9266,9 +9273,11 @@ object Http4s400 { else "" _ <- code.util.Helper.booleanToFuture(errorMsg, cc = callContext) { connectorMethod.isDefined } _ = Validation.validateDependency(connectorMethod.head) - (updated, _) <- NewStyle.function.updateJsonConnectorMethod( - connectorMethodId, connectorMethodBody.methodBody, connectorMethodBody.programmingLang, callContext) - } yield updated + result <- interceptOrApply(CONNECTOR_METHOD, ChangeOp.UPDATE, Some(connectorMethodId), 200, cc) { + NewStyle.function.updateJsonConnectorMethod( + connectorMethodId, connectorMethodBody.methodBody, connectorMethodBody.programmingLang, callContext).map(_._1) + } + } yield result } } @@ -9429,7 +9438,22 @@ object Http4s400 { } } - private def createDynamicResourceDocImpl(bankId: Option[String], rawBody: String, cc: CallContext): Future[JsonDynamicResourceDoc] = { + /** + * Maker/checker interception (MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md): after the body has been + * validated and compiled exactly as before, a managed target type is queued as a + * DynamicChangeRequest and answered with 202 instead of being applied. + */ + private def interceptOrApply[A](targetType: com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.Value, + operation: com.openbankproject.commons.model.enums.DynamicChangeRequestOperation.Value, + targetId: Option[String], appliedStatus: Int, cc: CallContext)(apply: => Future[A]): Future[(Any, Int)] = + Future(MakerChecker.intercept(targetType, operation, targetId, cc)) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestApprovalRequired, 400)) + .flatMap { + case Some(changeRequest) => Future.successful((createDynamicChangeRequestJsonV700(changeRequest), 202)) + case None => apply.map(result => (result, appliedStatus)) + } + + private def createDynamicResourceDocImpl(bankId: Option[String], rawBody: String, cc: CallContext): Future[(Any, Int)] = { for { _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled } body <- NewStyle.function.tryons( @@ -9444,11 +9468,13 @@ object Http4s400 { _ <- code.util.Helper.booleanToFuture( s"$DynamicResourceDocAlreadyExists The combination of request_url(${body.requestUrl}) and request_verb(${body.requestVerb}) must be unique", cc = callContext) { !isExists } - (created, _) <- NewStyle.function.createJsonDynamicResourceDoc(bankId, body, callContext) - } yield created + result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.CREATE, None, 201, cc) { + NewStyle.function.createJsonDynamicResourceDoc(bankId, body, callContext).map(_._1) + } + } yield result } - private def updateDynamicResourceDocImpl(bankId: Option[String], dynamicResourceDocId: String, rawBody: String, cc: CallContext): Future[JsonDynamicResourceDoc] = { + private def updateDynamicResourceDocImpl(bankId: Option[String], dynamicResourceDocId: String, rawBody: String, cc: CallContext): Future[(Any, Int)] = { for { _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled } body <- NewStyle.function.tryons( @@ -9459,14 +9485,25 @@ object Http4s400 { _ <- validateDynamicResourceDocBody(body, cc) _ = compileDynamicResourceDoc(body, cc) (_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(bankId, dynamicResourceDocId, Some(cc)) - (updated, _) <- NewStyle.function.updateJsonDynamicResourceDoc( - bankId, body.copy(dynamicResourceDocId = Some(dynamicResourceDocId)), callContext) - } yield updated + result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.UPDATE, Some(dynamicResourceDocId), 200, cc) { + NewStyle.function.updateJsonDynamicResourceDoc( + bankId, body.copy(dynamicResourceDocId = Some(dynamicResourceDocId)), callContext).map(_._1) + } + } yield result + } + + private def deleteDynamicResourceDocImpl(bankId: Option[String], dynamicResourceDocId: String, cc: CallContext): Future[(Any, Int)] = { + for { + (_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(bankId, dynamicResourceDocId, Some(cc)) + result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.DELETE, Some(dynamicResourceDocId), 204, cc) { + NewStyle.function.deleteJsonDynamicResourceDocById(bankId, dynamicResourceDocId, callContext).map(_._1) + } + } yield result } lazy val createDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "dynamic-resource-docs" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { val cc = req.callContext createDynamicResourceDocImpl(None, cc.httpBody.getOrElse(""), cc) } @@ -9474,18 +9511,16 @@ object Http4s400 { lazy val updateDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "dynamic-resource-docs" / dynamicResourceDocId => - EndpointHelpers.executeAndRespond(req) { cc => + EndpointHelpers.executeFutureWithStatus(req) { + val cc = req.callContext updateDynamicResourceDocImpl(None, dynamicResourceDocId, cc.httpBody.getOrElse(""), cc) } } lazy val deleteDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ DELETE -> `prefixPath` / "management" / "dynamic-resource-docs" / dynamicResourceDocId => - EndpointHelpers.withUserDelete(req) { (_, cc) => - for { - (_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(None, dynamicResourceDocId, Some(cc)) - (deleted, _) <- NewStyle.function.deleteJsonDynamicResourceDocById(None, dynamicResourceDocId, callContext) - } yield deleted + EndpointHelpers.withUserAndStatus(req) { (_, cc) => + deleteDynamicResourceDocImpl(None, dynamicResourceDocId, cc) } } @@ -9509,7 +9544,7 @@ object Http4s400 { lazy val createBankLevelDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-resource-docs" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { val cc = req.callContext createDynamicResourceDocImpl(Some(bankIdStr), cc.httpBody.getOrElse(""), cc) } @@ -9517,18 +9552,16 @@ object Http4s400 { lazy val updateBankLevelDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-resource-docs" / dynamicResourceDocId => - EndpointHelpers.executeAndRespond(req) { cc => + EndpointHelpers.executeFutureWithStatus(req) { + val cc = req.callContext updateDynamicResourceDocImpl(Some(bankIdStr), dynamicResourceDocId, cc.httpBody.getOrElse(""), cc) } } lazy val deleteBankLevelDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ DELETE -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-resource-docs" / dynamicResourceDocId => - EndpointHelpers.withUserDelete(req) { (_, cc) => - for { - (_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(Some(bankIdStr), dynamicResourceDocId, Some(cc)) - (deleted, _) <- NewStyle.function.deleteJsonDynamicResourceDocById(Some(bankIdStr), dynamicResourceDocId, callContext) - } yield deleted + EndpointHelpers.withUserAndStatus(req) { (_, cc) => + deleteDynamicResourceDocImpl(Some(bankIdStr), dynamicResourceDocId, cc) } } @@ -9729,7 +9762,7 @@ object Http4s400 { // Batch 17 — Dynamic Message Doc CRUD (system + bank level) // ═══════════════════════════════════════════════════════════════════════════ - private def createDynamicMessageDocImpl(bankId: Option[String], rawBody: String, cc: CallContext): Future[JsonDynamicMessageDoc] = { + private def createDynamicMessageDocImpl(bankId: Option[String], rawBody: String, cc: CallContext): Future[(Any, Int)] = { for { _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled } body <- NewStyle.function.tryons( @@ -9748,11 +9781,13 @@ object Http4s400 { else "" _ <- code.util.Helper.booleanToFuture(errorMsg, cc = callContext) { connectorMethod.isDefined } _ = Validation.validateDependency(connectorMethod.orNull) - (created, _) <- NewStyle.function.createJsonDynamicMessageDoc(bankId, body, callContext) - } yield created + result <- interceptOrApply(DYNAMIC_MESSAGE_DOC, ChangeOp.CREATE, None, 201, cc) { + NewStyle.function.createJsonDynamicMessageDoc(bankId, body, callContext).map(_._1) + } + } yield result } - private def updateDynamicMessageDocImpl(bankId: Option[String], dynamicMessageDocId: String, rawBody: String, cc: CallContext): Future[JsonDynamicMessageDoc] = { + private def updateDynamicMessageDocImpl(bankId: Option[String], dynamicMessageDocId: String, rawBody: String, cc: CallContext): Future[(Any, Int)] = { for { _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled } body <- NewStyle.function.tryons( @@ -9768,14 +9803,25 @@ object Http4s400 { _ <- code.util.Helper.booleanToFuture(errorMsg, cc = Some(cc)) { connectorMethod.isDefined } _ = Validation.validateDependency(connectorMethod.orNull) (_, callContext) <- NewStyle.function.getJsonDynamicMessageDocById(bankId, dynamicMessageDocId, Some(cc)) - (updated, _) <- NewStyle.function.updateJsonDynamicMessageDoc( - bankId, body.copy(dynamicMessageDocId = Some(dynamicMessageDocId)), callContext) - } yield updated + result <- interceptOrApply(DYNAMIC_MESSAGE_DOC, ChangeOp.UPDATE, Some(dynamicMessageDocId), 200, cc) { + NewStyle.function.updateJsonDynamicMessageDoc( + bankId, body.copy(dynamicMessageDocId = Some(dynamicMessageDocId)), callContext).map(_._1) + } + } yield result + } + + private def deleteDynamicMessageDocImpl(bankId: Option[String], dynamicMessageDocId: String, cc: CallContext): Future[(Any, Int)] = { + for { + (_, callContext) <- NewStyle.function.getJsonDynamicMessageDocById(bankId, dynamicMessageDocId, Some(cc)) + result <- interceptOrApply(DYNAMIC_MESSAGE_DOC, ChangeOp.DELETE, Some(dynamicMessageDocId), 204, cc) { + NewStyle.function.deleteJsonDynamicMessageDocById(bankId, dynamicMessageDocId, callContext).map(_._1) + } + } yield result } lazy val createDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "dynamic-message-docs" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { val cc = req.callContext createDynamicMessageDocImpl(None, cc.httpBody.getOrElse(""), cc) } @@ -9783,18 +9829,16 @@ object Http4s400 { lazy val updateDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "dynamic-message-docs" / dynamicMessageDocId => - EndpointHelpers.executeAndRespond(req) { cc => + EndpointHelpers.executeFutureWithStatus(req) { + val cc = req.callContext updateDynamicMessageDocImpl(None, dynamicMessageDocId, cc.httpBody.getOrElse(""), cc) } } lazy val deleteDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ DELETE -> `prefixPath` / "management" / "dynamic-message-docs" / dynamicMessageDocId => - EndpointHelpers.withUserDelete(req) { (_, cc) => - for { - (_, callContext) <- NewStyle.function.getJsonDynamicMessageDocById(None, dynamicMessageDocId, Some(cc)) - (deleted, _) <- NewStyle.function.deleteJsonDynamicMessageDocById(None, dynamicMessageDocId, callContext) - } yield deleted + EndpointHelpers.withUserAndStatus(req) { (_, cc) => + deleteDynamicMessageDocImpl(None, dynamicMessageDocId, cc) } } @@ -9818,7 +9862,7 @@ object Http4s400 { lazy val createBankLevelDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-message-docs" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { val cc = req.callContext createDynamicMessageDocImpl(Some(bankIdStr), cc.httpBody.getOrElse(""), cc) } @@ -9826,18 +9870,16 @@ object Http4s400 { lazy val updateBankLevelDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-message-docs" / dynamicMessageDocId => - EndpointHelpers.executeAndRespond(req) { cc => + EndpointHelpers.executeFutureWithStatus(req) { + val cc = req.callContext updateDynamicMessageDocImpl(Some(bankIdStr), dynamicMessageDocId, cc.httpBody.getOrElse(""), cc) } } lazy val deleteBankLevelDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ DELETE -> `prefixPath` / "management" / "banks" / bankIdStr / "dynamic-message-docs" / dynamicMessageDocId => - EndpointHelpers.withUserDelete(req) { (_, cc) => - for { - (_, callContext) <- NewStyle.function.getJsonDynamicMessageDocById(Some(bankIdStr), dynamicMessageDocId, Some(cc)) - (deleted, _) <- NewStyle.function.deleteJsonDynamicMessageDocById(Some(bankIdStr), dynamicMessageDocId, callContext) - } yield deleted + EndpointHelpers.withUserAndStatus(req) { (_, cc) => + deleteDynamicMessageDocImpl(Some(bankIdStr), dynamicMessageDocId, cc) } } diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index c98fdfa8bd..4497c5e2ca 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -25,6 +25,8 @@ import code.api.util.APIUtil.{ import code.api.util.{ExampleValue, Glossary} import code.api.v1_2_1.{AccountHolderJSON, BankRoutingJsonV121, TransactionDetailsJSON} import code.api.v4_0_0.BankAttributeBankResponseJsonV400 +import code.dynamicchangerequest.MakerChecker +import code.api.v7_0_0.JSONFactory700.createDynamicChangeRequestJsonV700 import code.bankconnectors.LocalMappedConnectorInternal.transactionRequestGeneralText import code.webuiprops.WebUiPropsPutJsonV600 import com.openbankproject.commons.model.{ @@ -2859,22 +2861,8 @@ object Http4s600 { lazy val getSignalChannels: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> `prefixPath` / "signal" / "channels" => EndpointHelpers.withUser(req) { (_, cc) => - Future { - val names = code.api.cache.RedisMessaging.listChannels() - val infos = names.flatMap { name => - code.api.cache.RedisMessaging.channelInfo(name).map { case (count, ttl) => - val (messages, _) = code.api.cache.RedisMessaging.fetchMessages(name, 0, count.toInt) - val hasBroadcast = messages.exists { s => - scala.util.Try(com.openbankproject.commons.util.JsonAliases.parse(s).extract[SignalMessageJsonV600].to_user_id.isEmpty).getOrElse(false) - } - (name, count, ttl, hasBroadcast) - } - } - val channels = infos.filter(_._4).map { case (name, count, ttl, _) => - SignalChannelInfoJsonV600(name, count, ttl) - } - SignalChannelsJsonV600(channels) - } + // Shared with the gRPC SignalChannelsService.ListChannels, see code.signal.SignalChannels + Future(SignalChannelsJsonV600(code.signal.SignalChannels.listBroadcastChannels())) } } @@ -2945,21 +2933,10 @@ object Http4s600 { !code.signal.SignalContentPolicy.containsDangerousCharacters(postJson.payload) && postJson.message_type.forall(messageType => !code.util.DangerousCharacters.containsAny(messageType)) } + // Envelope building and storage are shared with the gRPC SignalChannelsService.Publish published <- Future { val consumerId = cc.consumer match { case Full(c) => c.consumerId.get; case _ => "" } - val messageId = randomUUID().toString - val sdf = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") - sdf.setTimeZone(java.util.TimeZone.getTimeZone("UTC")) - val timestamp = sdf.format(new java.util.Date()) - val envelope = SignalMessageJsonV600( - message_id = messageId, channel_name = channelName, - sender_consumer_id = consumerId, sender_user_id = u.userId, - to_user_id = postJson.to_user_id, timestamp = timestamp, - message_type = postJson.message_type.getOrElse(""), - payload = postJson.payload) - val msgStr = com.openbankproject.commons.util.JsonAliases.compactRender(Extraction.decompose(envelope)) - val count = code.api.cache.RedisMessaging.publishMessage(channelName, msgStr) - SignalMessagePublishedJsonV600(messageId, channelName, timestamp, count) + code.signal.SignalChannels.publish(channelName, u.userId, consumerId, postJson) } } yield published } @@ -2977,18 +2954,18 @@ object Http4s600 { (obpQueryParams, _) <- createQueriesByHttpParamsFuture(httpParams, Some(cc)) limit = obpQueryParams.collectFirst { case code.api.util.OBPLimit(value) => value }.getOrElse(50) offset = obpQueryParams.collectFirst { case code.api.util.OBPOffset(value) => value }.getOrElse(0) - (rawMessages, totalCount) <- Future(code.api.cache.RedisMessaging.fetchMessages(channelName, offset, limit)) - } yield { - val parsed = rawMessages.flatMap { s => - scala.util.Try(com.openbankproject.commons.util.JsonAliases.parse(s).extract[SignalMessageJsonV600]).toOption - } - val filtered = parsed.filter { msg => - msg.to_user_id.isEmpty || - msg.to_user_id.contains(user.userId) || - msg.sender_user_id == user.userId + // after_sequence switches to a cursor read that survives the channel being trimmed; + // offset paging is kept for browsing what the channel holds right now. + afterSequenceParam = req.uri.query.params.get("after_sequence").map(_.trim).filter(_.nonEmpty) + afterSequence <- afterSequenceParam match { + case None => Future.successful(None) + case Some(raw) => NewStyle.function.tryons(s"$InvalidNumber after_sequence must be an integer, got: $raw", 400, Some(cc)) { + Some(raw.toLong) + } } - SignalMessagesJsonV600(channelName, filtered, totalCount, (offset + limit) < totalCount) - } + // Storage access and privacy filter shared with the gRPC SignalChannelsService.Fetch + page <- Future(code.signal.SignalChannels.fetch(channelName, offset, limit, afterSequence, user.userId)) + } yield page } } @@ -5882,7 +5859,7 @@ object Http4s600 { // Route: POST /obp/v6.0.0/management/abac-rules (201) lazy val createAbacRule: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "abac-rules" => - EndpointHelpers.executeFutureCreated(req) { + EndpointHelpers.executeFutureWithStatus(req) { implicit val cc: CallContext = req.callContext val rawBody = cc.httpBody.getOrElse("") val user = cc.user.openOrThrowException(AuthenticatedUserIsRequired) @@ -5895,12 +5872,20 @@ object Http4s600 { _ <- Helper.booleanToFuture("Rule code must not be empty", cc = Some(cc)) { createJson.rule_code.nonEmpty } _ <- AbacRuleEngine.validateRuleCodeAsync(createJson.rule_code) .map(unboxFullOrFail(_, Some(cc), "Invalid ABAC rule code", 400)) - rule <- Future(MappedAbacRuleProvider.createAbacRule( - ruleName = createJson.rule_name, ruleCode = createJson.rule_code, - description = createJson.description, policy = createJson.policy, - isActive = createJson.is_active, createdBy = user.userId - )).map(unboxFullOrFail(_, Some(cc), "Could not create ABAC rule", 400)) - } yield createAbacRuleJsonV600(rule) + // Maker/checker: a managed instance queues the rule for a second user's approval (202) + intercepted <- Future(MakerChecker.intercept( + com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.ABAC_RULE, + com.openbankproject.commons.model.enums.DynamicChangeRequestOperation.CREATE, None, cc)) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestApprovalRequired, 400)) + result <- intercepted match { + case Some(changeRequest) => Future.successful((createDynamicChangeRequestJsonV700(changeRequest), 202)) + case None => Future(MappedAbacRuleProvider.createAbacRule( + ruleName = createJson.rule_name, ruleCode = createJson.rule_code, + description = createJson.description, policy = createJson.policy, + isActive = createJson.is_active, createdBy = user.userId + )).map(unboxFullOrFail(_, Some(cc), "Could not create ABAC rule", 400)).map(rule => (createAbacRuleJsonV600(rule), 201)) + } + } yield result } } @@ -5938,7 +5923,8 @@ object Http4s600 { // Route: PUT /obp/v6.0.0/management/abac-rules/ABAC_RULE_ID lazy val updateAbacRule: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "management" / "abac-rules" / ruleId => - EndpointHelpers.executeAndRespond(req) { implicit cc => + EndpointHelpers.executeFutureWithStatus(req) { + implicit val cc: CallContext = req.callContext val rawBody = cc.httpBody.getOrElse("") val user = cc.user.openOrThrowException(AuthenticatedUserIsRequired) for { @@ -5948,14 +5934,25 @@ object Http4s600 { } _ <- AbacRuleEngine.validateRuleCodeAsync(updateJson.rule_code) .map(unboxFullOrFail(_, Some(cc), "Invalid ABAC rule code", 400)) - rule <- Future(MappedAbacRuleProvider.updateAbacRule( - ruleId = ruleId, ruleName = updateJson.rule_name, - ruleCode = updateJson.rule_code, description = updateJson.description, - policy = updateJson.policy, isActive = updateJson.is_active, - updatedBy = user.userId - )).map(unboxFullOrFail(_, Some(cc), s"Could not update ABAC rule with ID: $ruleId", 400)) - _ <- Future(AbacRuleEngine.clearRuleFromCache(ruleId)) - } yield createAbacRuleJsonV600(rule) + _ <- Future(MappedAbacRuleProvider.getAbacRuleById(ruleId)) + .map(unboxFullOrFail(_, Some(cc), s"ABAC Rule not found with ID: $ruleId", 404)) + intercepted <- Future(MakerChecker.intercept( + com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.ABAC_RULE, + com.openbankproject.commons.model.enums.DynamicChangeRequestOperation.UPDATE, Some(ruleId), cc)) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestApprovalRequired, 400)) + result <- intercepted match { + case Some(changeRequest) => Future.successful((createDynamicChangeRequestJsonV700(changeRequest), 202)) + case None => for { + rule <- Future(MappedAbacRuleProvider.updateAbacRule( + ruleId = ruleId, ruleName = updateJson.rule_name, + ruleCode = updateJson.rule_code, description = updateJson.description, + policy = updateJson.policy, isActive = updateJson.is_active, + updatedBy = user.userId + )).map(unboxFullOrFail(_, Some(cc), s"Could not update ABAC rule with ID: $ruleId", 400)) + _ <- Future(AbacRuleEngine.clearRuleFromCache(ruleId)) + } yield (createAbacRuleJsonV600(rule), 200) + } + } yield result } } @@ -5963,12 +5960,24 @@ object Http4s600 { // Route: DELETE /obp/v6.0.0/management/abac-rules/ABAC_RULE_ID lazy val deleteAbacRule: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ DELETE -> `prefixPath` / "management" / "abac-rules" / ruleId => - EndpointHelpers.executeAndRespond(req) { implicit cc => + EndpointHelpers.executeFutureWithStatus(req) { + implicit val cc: CallContext = req.callContext for { - _ <- Future(MappedAbacRuleProvider.deleteAbacRule(ruleId)) + _ <- Future(MappedAbacRuleProvider.getAbacRuleById(ruleId)) .map(unboxFullOrFail(_, Some(cc), s"Could not delete ABAC rule with ID: $ruleId", 400)) - _ <- Future(AbacRuleEngine.clearRuleFromCache(ruleId)) - } yield "" + intercepted <- Future(MakerChecker.intercept( + com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.ABAC_RULE, + com.openbankproject.commons.model.enums.DynamicChangeRequestOperation.DELETE, Some(ruleId), cc)) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestApprovalRequired, 400)) + result <- intercepted match { + case Some(changeRequest) => Future.successful((createDynamicChangeRequestJsonV700(changeRequest), 202)) + case None => for { + _ <- Future(MappedAbacRuleProvider.deleteAbacRule(ruleId)) + .map(unboxFullOrFail(_, Some(cc), s"Could not delete ABAC rule with ID: $ruleId", 400)) + _ <- Future(AbacRuleEngine.clearRuleFromCache(ruleId)) + } yield ("", 200) + } + } yield result } } @@ -10488,7 +10497,10 @@ object Http4s600 { |AI agents and other OBP consumers. Messages are not persisted to a database. | |Channels are auto-created on first publish and expire after a configurable TTL (default 1 hour). - |Messages are capped at a configurable maximum per channel (default 1000). + |Messages are capped at a configurable maximum per channel (default 1000); the oldest are trimmed. + | + |The response carries the per-channel monotonic `sequence` stamped on the stored message. Readers + |poll with `after_sequence` on Get Signal Messages, which is unaffected by trimming. | |The payload field accepts any valid JSON content. On this instance the whole request body |may be up to ${code.signal.SignalContentPolicy.maxPayloadLength} characters. @@ -10500,6 +10512,10 @@ object Http4s600 { |Set to_user_id to send a private message visible only to the sender and recipient. |Leave to_user_id empty for a broadcast message visible to all channel readers. | + |Live delivery: every publish is also pushed to gRPC clients streaming the channel via + |SignalChannelsService.Subscribe (see signal.proto). The same service offers Publish, Fetch + |and ListChannels as 1:1 equivalents of the REST endpoints, over the same Redis storage. + | |Authentication is Required. | |""".stripMargin, @@ -10531,19 +10547,28 @@ object Http4s600 { |Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery |and coordination, but usable by any authenticated OBP consumer. | - |Messages are returned oldest-first. + |Messages are returned oldest-first. Every message carries a per-channel monotonic + |**sequence** stamped when it was published. | |Privacy filtering is applied server-side: you will only see broadcast messages (no to_user_id) |and private messages addressed to you (to_user_id matches your user ID) or sent by you. | - |Use the offset parameter to poll for new messages by tracking your position. + |**To poll, use `after_sequence`**, not offset: `?after_sequence=&limit=50` + |returns only newer messages. A channel keeps just its newest messages (default 1000) and trims + |the oldest, which moves list positions, so a poller that tracks an offset silently skips messages + |once the channel fills. The response's `next_after_sequence` is the value to send next; it + |advances even when every message in the window was private to someone else, and `has_more` + |says whether newer messages remain. `latest_sequence` is the newest message in the channel. + |Start with `after_sequence=0` for everything the channel still holds. + | + |Without `after_sequence`, offset and limit page over the channel's current contents. | |Authentication is Required. | |""".stripMargin, EmptyBody, signalMessagesJsonV600, - List($AuthenticatedUserIsRequired, InvalidSignalChannelName, UnknownError), + List($AuthenticatedUserIsRequired, InvalidSignalChannelName, InvalidNumber, UnknownError), apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, None, http4sPartialFunction = Some(getSignalMessages) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 364de47d39..06e53e9159 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -1205,6 +1205,11 @@ case class PostSignalMessageJsonV600( case class SignalMessageJsonV600( message_id: String, + // Per-channel monotonic sequence stamped atomically at publish (Redis server time in + // microseconds, forced strictly increasing). Poll with after_sequence=, never by + // offset: the channel list is trimmed to its newest N messages, which shifts list indexes. + // Defaulted so envelopes stored before this field existed still parse (they read as 0). + sequence: Long = 0L, channel_name: String, sender_consumer_id: String, sender_user_id: String, @@ -1218,14 +1223,20 @@ case class SignalMessagesJsonV600( channel_name: String, messages: List[SignalMessageJsonV600], total_count: Long, - has_more: Boolean + has_more: Boolean, + // Sequence of the newest message in the channel (0 when empty). + latest_sequence: Long, + // Pass this back as after_sequence to continue. It advances past messages the privacy + // filter hid from you, so a page can be empty and the cursor still moves. + next_after_sequence: Long ) case class SignalMessagePublishedJsonV600( message_id: String, channel_name: String, timestamp: String, - channel_message_count: Long + channel_message_count: Long, + sequence: Long ) case class SignalChannelInfoJsonV600( diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 3226fb0733..836d6c67b9 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -5964,6 +5964,421 @@ object Http4s700 { http4sPartialFunction = Some(deleteApiProductSubscriptionAttribute) ).disableAutoValidateRoles() + // ═══════════════════════════════════════════════════════════════════════════ + // Maker/checker for dynamic code: Dynamic Change Requests + // Design: MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md. Approval is system level (no bank endpoints). + // ═══════════════════════════════════════════════════════════════════════════ + + import code.dynamicchangerequest.{DynamicChangeRequestTrait, MakerChecker} + import code.api.v7_0_0.JSONFactory700.{DynamicChangeRequestJsonV700, DynamicChangeRequestsJsonV700, PostApproveDynamicChangeRequestJsonV700, PostDeactivateDynamicArtefactJsonV700, PostDynamicChangeRequestJsonV700, PostRejectDynamicChangeRequestJsonV700, PostWithdrawDynamicChangeRequestJsonV700} + import com.openbankproject.commons.model.enums.{DynamicChangeRequestOperation, DynamicChangeRequestTargetType} + import com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType._ + + private def changeRequestProvider = DynamicChangeRequestTrait.dynamicChangeRequest.vend + + private def loadChangeRequest(id: String, cc: CallContext): Future[DynamicChangeRequestTrait] = Future { + unboxFullOrFail(changeRequestProvider.getById(id), Some(cc), s"$DynamicChangeRequestNotFound CHANGE_REQUEST_ID($id)", 404) + }.map(MakerChecker.expireIfDue) + + /** The maker must already hold the role the direct v4/v6 write would demand. */ + private def makerRolesFor(targetType: DynamicChangeRequestTargetType, operation: DynamicChangeRequestOperation.Value, bankLevel: Boolean): List[ApiRole] = + (targetType, operation) match { + case (DYNAMIC_RESOURCE_DOC, DynamicChangeRequestOperation.CREATE) => if (bankLevel) List(ApiRole.canCreateBankLevelDynamicResourceDoc) else List(ApiRole.canCreateDynamicResourceDoc) + case (DYNAMIC_RESOURCE_DOC, DynamicChangeRequestOperation.UPDATE | DynamicChangeRequestOperation.ACTIVATE) => if (bankLevel) List(ApiRole.canUpdateBankLevelDynamicResourceDoc) else List(ApiRole.canUpdateDynamicResourceDoc) + case (DYNAMIC_RESOURCE_DOC, DynamicChangeRequestOperation.DELETE) => if (bankLevel) List(ApiRole.canDeleteBankLevelDynamicResourceDoc) else List(ApiRole.canDeleteDynamicResourceDoc) + case (DYNAMIC_MESSAGE_DOC, DynamicChangeRequestOperation.CREATE) => if (bankLevel) List(ApiRole.canCreateBankLevelDynamicMessageDoc) else List(ApiRole.canCreateDynamicMessageDoc) + case (DYNAMIC_MESSAGE_DOC, DynamicChangeRequestOperation.UPDATE | DynamicChangeRequestOperation.ACTIVATE) => List(ApiRole.canUpdateDynamicMessageDoc) + case (DYNAMIC_MESSAGE_DOC, DynamicChangeRequestOperation.DELETE) => if (bankLevel) List(ApiRole.canDeleteBankLevelDynamicMessageDoc) else List(ApiRole.canDeleteDynamicMessageDoc) + case (CONNECTOR_METHOD, DynamicChangeRequestOperation.CREATE) => List(ApiRole.canCreateConnectorMethod) + case (CONNECTOR_METHOD, _) => List(ApiRole.canUpdateConnectorMethod) + case (ABAC_RULE, DynamicChangeRequestOperation.CREATE) => List(ApiRole.canCreateAbacRule) + case (ABAC_RULE, DynamicChangeRequestOperation.UPDATE | DynamicChangeRequestOperation.ACTIVATE) => List(ApiRole.canUpdateAbacRule) + case (ABAC_RULE, DynamicChangeRequestOperation.DELETE) => List(ApiRole.canDeleteAbacRule) + case _ => List(ApiRole.canApproveDynamicChangeRequest) + } + + /** The v4/v6 path the explicit submission stands in for, so apply() resolves the same scope. */ + private def syntheticRequestPath(targetType: DynamicChangeRequestTargetType, targetId: Option[String], bankId: Option[String]): (String, String) = { + val (version, segment) = targetType match { + case DYNAMIC_RESOURCE_DOC => ("v4.0.0", "dynamic-resource-docs") + case DYNAMIC_MESSAGE_DOC => ("v4.0.0", "dynamic-message-docs") + case CONNECTOR_METHOD => ("v4.0.0", "connector-methods") + case ABAC_RULE => ("v6.0.0", "abac-rules") + case other => ("v7.0.0", other.toString.toLowerCase.replace('_', '-')) + } + val scope = bankId.filter(_.nonEmpty).map(b => s"/banks/$b").getOrElse("") + val id = targetId.filter(_.nonEmpty).map("/" + _).getOrElse("") + (version, s"/obp/$version/management$scope/$segment$id") + } + + private val changeRequestExample = DynamicChangeRequestJsonV700( + dynamic_change_request_id = "0d1c9e3c-6c2b-4c1e-9a53-2d5b2f0d7f11", + target_type = "DYNAMIC_RESOURCE_DOC", + target_id = "", + operation = "CREATE", + status = "INITIATED", + request_verb = "POST", + request_path = "/obp/v4.0.0/management/dynamic-resource-docs", + payload_hash = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + current_payload_hash = "", + proposed_payload = Extraction.decompose(jsonDynamicResourceDoc), + current_payload = JNothing, + requestor_user_id = code.api.util.ExampleValue.userIdExample.value, + business_justification = "Expose the new loan quote endpoint for the mobile app.", + checker_user_id = "", + checker_comment = "", + created_at = APIUtil.DateWithMsExampleString, + actioned_at = "", + expires_at = APIUtil.DateWithMsExampleString + ) + + private val makerCheckerIntro = + s"""Maker/checker for runtime-supplied code and configuration (dynamic resource docs, connector methods, dynamic message docs, ABAC rules). + | + |When `dynamic_code_requires_approval` is true and the target type is listed in `dynamic_code_approval_target_types`, the v4.0.0 / v6.0.0 create, update and delete endpoints validate and compile the body as before but return `202 Accepted` with a Dynamic Change Request instead of applying it. A DIFFERENT user holding `CanApproveDynamicChangeRequest` approves the request by its `payload_hash`; only then is the change applied, and the runtime executes only rows whose body hash equals the approved hash. + | + |Approval is system level. Dynamic code runs in the shared JVM, so a bank-level artefact is approved by the same system-level checker; there are no bank-level change request endpoints. + |""".stripMargin + + val createDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-change-requests" => + EndpointHelpers.executeFutureCreated(req) { + implicit val cc: CallContext = req.callContext + val u = cc.user.openOrThrowException(AuthenticatedUserIsRequired) + val rawBody = cc.httpBody.getOrElse("") + for { + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostDynamicChangeRequestJsonV700", 400, Some(cc)) { + com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[PostDynamicChangeRequestJsonV700] + } + targetType <- NewStyle.function.tryons(s"$InvalidJsonFormat target_type must be one of ${DynamicChangeRequestTargetType.values.mkString(", ")}", 400, Some(cc)) { + DynamicChangeRequestTargetType.withName(postJson.target_type) + } + operation <- NewStyle.function.tryons(s"$InvalidJsonFormat operation must be one of CREATE, UPDATE, DELETE, ACTIVATE", 400, Some(cc)) { + DynamicChangeRequestOperation.withName(postJson.operation) + } + _ <- Helper.booleanToFuture(s"$InvalidJsonFormat operation DEACTIVATE is a direct action, use the deactivation endpoint", cc = Some(cc)) { operation != DynamicChangeRequestOperation.DEACTIVATE } + _ <- Helper.booleanToFuture(s"$DynamicChangeRequestTargetTypeNotManaged ${postJson.target_type}", cc = Some(cc)) { MakerChecker.isManaged(targetType) } + _ <- Helper.booleanToFuture(s"$InvalidJsonFormat target_id is required for ${operation}", cc = Some(cc)) { + operation == DynamicChangeRequestOperation.CREATE || postJson.target_id.exists(_.nonEmpty) + } + bankLevel = postJson.bank_id.exists(_.nonEmpty) + roles = makerRolesFor(targetType, operation, bankLevel) + _ <- Helper.booleanToFuture(UserHasMissingRoles + roles.mkString(" or "), failCode = 403, cc = Some(cc)) { + APIUtil.hasAtLeastOneEntitlement(postJson.bank_id.getOrElse(""), u.userId, roles) + } + (verb, path) = { + val (_, p) = syntheticRequestPath(targetType, postJson.target_id, postJson.bank_id) + (operation match { case DynamicChangeRequestOperation.CREATE => "POST"; case DynamicChangeRequestOperation.UPDATE => "PUT"; case DynamicChangeRequestOperation.DELETE => "DELETE"; case _ => "POST" }, p) + } + payload = if (operation == DynamicChangeRequestOperation.ACTIVATE) """{"is_active":true}""" else com.openbankproject.commons.util.JsonAliases.compactRender(postJson.proposed_payload) + created <- Future(MakerChecker.submit(targetType, operation, postJson.target_id, verb, path, payload, u.userId, postJson.business_justification.getOrElse(""))) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestTargetNotFound, 404)) + } yield JSONFactory700.createDynamicChangeRequestJsonV700(created) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(createDynamicChangeRequest), + "POST", + "/management/dynamic-change-requests", + "Create Dynamic Change Request", + s"""Submit a change to a dynamic artefact for approval by a second user. + | + |$makerCheckerIntro + |This explicit submission is for tooling that wants to attach a `business_justification` up front. In the common case the maker simply calls the v4.0.0 / v6.0.0 create, update or delete endpoint and receives the change request in a `202 Accepted` response. + | + |`proposed_payload` is the exact body the corresponding v4.0.0 / v6.0.0 endpoint accepts. `bank_id` selects the bank-level variant of that endpoint; omit it for system level. `target_id` is required for UPDATE, DELETE and ACTIVATE. The caller must hold the role the direct write would demand (for example `CanCreateDynamicResourceDoc`). + | + |${userAuthenticationMessage(true)}""".stripMargin, + PostDynamicChangeRequestJsonV700("DYNAMIC_RESOURCE_DOC", "CREATE", None, None, Extraction.decompose(jsonDynamicResourceDoc), Some("Expose the new loan quote endpoint for the mobile app.")), + changeRequestExample, + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicChangeRequestTargetTypeNotManaged, DynamicChangeRequestTargetNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + None, + http4sPartialFunction = Some(createDynamicChangeRequest) + ) + + val getDynamicChangeRequests: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "dynamic-change-requests" => + EndpointHelpers.withUser(req) { (_, cc) => + val q = req.uri.query.multiParams + def param(name: String): Option[String] = q.get(name).flatMap(_.headOption).filter(_.nonEmpty) + Future { + changeRequestProvider.getAll(param("status"), param("target_type"), param("target_id"), param("requestor_user_id")) + .map(MakerChecker.expireIfDue) + }.map(rows => DynamicChangeRequestsJsonV700(rows.map(JSONFactory700.createDynamicChangeRequestJsonV700))) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicChangeRequests), + "GET", + "/management/dynamic-change-requests", + "Get Dynamic Change Requests", + s"""Returns all Dynamic Change Requests, newest first. The table is never pruned: it is the audit log of every proposed, approved, rejected, withdrawn, expired and failed change. + | + |Optional query parameters: `status` (INITIATED, APPROVED, REJECTED, WITHDRAWN, EXPIRED, FAILED), `target_type`, `target_id`, `requestor_user_id`. + | + |$makerCheckerIntro + |${userAuthenticationMessage(true)}""".stripMargin, + EmptyBody, + DynamicChangeRequestsJsonV700(List(changeRequestExample)), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + Some(List(ApiRole.canGetDynamicChangeRequests)), + http4sPartialFunction = Some(getDynamicChangeRequests) + ) + + val getMyDynamicChangeRequests: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "my" / "dynamic-change-requests" => + EndpointHelpers.withUser(req) { (u, cc) => + Future(changeRequestProvider.getByRequestorUserId(u.userId).map(MakerChecker.expireIfDue)) + .map(rows => DynamicChangeRequestsJsonV700(rows.map(JSONFactory700.createDynamicChangeRequestJsonV700))) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getMyDynamicChangeRequests), + "GET", + "/my/dynamic-change-requests", + "Get My Dynamic Change Requests", + s"""Returns the Dynamic Change Requests submitted by the authenticated user, newest first. No role is required. + | + |${userAuthenticationMessage(true)}""".stripMargin, + EmptyBody, + DynamicChangeRequestsJsonV700(List(changeRequestExample)), + List($AuthenticatedUserIsRequired, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + None, + http4sPartialFunction = Some(getMyDynamicChangeRequests) + ) + + val getDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "dynamic-change-requests" / changeRequestId if changeRequestId.nonEmpty => + EndpointHelpers.withUser(req) { (_, cc) => + loadChangeRequest(changeRequestId, cc).map(JSONFactory700.createDynamicChangeRequestJsonV700) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicChangeRequest), + "GET", + "/management/dynamic-change-requests/CHANGE_REQUEST_ID", + "Get Dynamic Change Request", + s"""Returns one Dynamic Change Request: the proposed payload, the live target's current payload (so a client can render a diff), both hashes and the status. + | + |A checker approves by sending back `payload_hash` exactly as returned here. + | + |${userAuthenticationMessage(true)}""".stripMargin, + EmptyBody, + changeRequestExample, + List($AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicChangeRequestNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + Some(List(ApiRole.canGetDynamicChangeRequests)), + http4sPartialFunction = Some(getDynamicChangeRequest) + ) + + val approveDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-change-requests" / changeRequestId / "approval" if changeRequestId.nonEmpty => + EndpointHelpers.withUser(req) { (u, cc) => + implicit val c: CallContext = cc + val rawBody = cc.httpBody.getOrElse("") + for { + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostApproveDynamicChangeRequestJsonV700", 400, Some(cc)) { + com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[PostApproveDynamicChangeRequestJsonV700] + } + request <- loadChangeRequest(changeRequestId, cc) + approved <- Future(MakerChecker.approve(request, u.userId, postJson.payload_hash, postJson.checker_comment.getOrElse(""))) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestNotInitiated, 400)) + } yield JSONFactory700.createDynamicChangeRequestJsonV700(approved) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(approveDynamicChangeRequest), + "POST", + "/management/dynamic-change-requests/CHANGE_REQUEST_ID/approval", + "Approve Dynamic Change Request", + s"""Approve a Dynamic Change Request and apply it. + | + |The checker must not be the requestor ($MakerCheckerSameUser). `payload_hash` must equal the request's stored hash, so the client has to show the checker exactly what is being approved. The server then re-checks that the live target has not changed since submission, wins the INITIATED to APPROVED transition (a concurrent approval or rejection loses), re-compiles and re-validates the payload, applies it through the same provider the v4.0.0 / v6.0.0 endpoint uses, and records the approved body hash on the target. If re-validation or apply fails the request is moved to FAILED with the reason in `checker_comment` and the target is left unchanged. + | + |${userAuthenticationMessage(true)}""".stripMargin, + PostApproveDynamicChangeRequestJsonV700("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", Some("Reviewed the method body; matches UAT hash.")), + changeRequestExample.copy(status = "APPROVED", checker_user_id = code.api.util.ExampleValue.userIdExample.value, checker_comment = "Reviewed the method body; matches UAT hash.", actioned_at = APIUtil.DateWithMsExampleString), + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicChangeRequestNotFound, DynamicChangeRequestNotInitiated, DynamicChangeRequestHashMismatch, DynamicChangeRequestStale, MakerCheckerSameUser, DynamicChangeRequestApplyFailed, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(approveDynamicChangeRequest) + ) + + val rejectDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-change-requests" / changeRequestId / "rejection" if changeRequestId.nonEmpty => + EndpointHelpers.withUser(req) { (u, cc) => + implicit val c: CallContext = cc + val rawBody = cc.httpBody.getOrElse("") + for { + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostRejectDynamicChangeRequestJsonV700", 400, Some(cc)) { + com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[PostRejectDynamicChangeRequestJsonV700] + } + _ <- Helper.booleanToFuture(CheckerCommentRequiredForRejection, cc = Some(cc)) { postJson.comment.trim.nonEmpty } + request <- loadChangeRequest(changeRequestId, cc) + rejected <- Future(MakerChecker.reject(request, u.userId, postJson.comment)) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestNotInitiated, 400)) + } yield JSONFactory700.createDynamicChangeRequestJsonV700(rejected) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(rejectDynamicChangeRequest), + "POST", + "/management/dynamic-change-requests/CHANGE_REQUEST_ID/rejection", + "Reject Dynamic Change Request", + s"""Reject a Dynamic Change Request. A comment is required. The checker must not be the requestor. Nothing is applied. + | + |${userAuthenticationMessage(true)}""".stripMargin, + PostRejectDynamicChangeRequestJsonV700("Body performs network calls that are not permitted."), + changeRequestExample.copy(status = "REJECTED", checker_user_id = code.api.util.ExampleValue.userIdExample.value, checker_comment = "Body performs network calls that are not permitted.", actioned_at = APIUtil.DateWithMsExampleString), + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicChangeRequestNotFound, DynamicChangeRequestNotInitiated, MakerCheckerSameUser, CheckerCommentRequiredForRejection, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(rejectDynamicChangeRequest) + ) + + val withdrawDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-change-requests" / changeRequestId / "withdrawal" if changeRequestId.nonEmpty => + EndpointHelpers.withUser(req) { (u, cc) => + implicit val c: CallContext = cc + val rawBody = cc.httpBody.getOrElse("") + for { + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostWithdrawDynamicChangeRequestJsonV700", 400, Some(cc)) { + if (rawBody.trim.isEmpty) PostWithdrawDynamicChangeRequestJsonV700(None) + else com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[PostWithdrawDynamicChangeRequestJsonV700] + } + request <- loadChangeRequest(changeRequestId, cc) + withdrawn <- Future(MakerChecker.withdraw(request, u.userId, postJson.comment.getOrElse(""))) + .map(unboxFullOrFail(_, Some(cc), DynamicChangeRequestNotInitiated, 400)) + } yield JSONFactory700.createDynamicChangeRequestJsonV700(withdrawn) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(withdrawDynamicChangeRequest), + "POST", + "/management/dynamic-change-requests/CHANGE_REQUEST_ID/withdrawal", + "Withdraw Dynamic Change Request", + s"""Withdraw a Dynamic Change Request you submitted. Only the requestor can withdraw, and only while it is INITIATED. No role is required. + | + |${userAuthenticationMessage(true)}""".stripMargin, + PostWithdrawDynamicChangeRequestJsonV700(Some("Superseded by a corrected version.")), + changeRequestExample.copy(status = "WITHDRAWN", checker_comment = "Superseded by a corrected version.", actioned_at = APIUtil.DateWithMsExampleString), + List($AuthenticatedUserIsRequired, InvalidJsonFormat, DynamicChangeRequestNotFound, DynamicChangeRequestNotInitiated, DynamicChangeRequestNotRequestor, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: Nil, + None, + http4sPartialFunction = Some(withdrawDynamicChangeRequest) + ) + + // ─── Deactivation: four eyes to enable, one pair to disable ─────────────── + private def deactivateArtefact(req: Request[IO], targetType: DynamicChangeRequestTargetType, targetId: String): IO[Response[IO]] = + EndpointHelpers.withUser(req) { (u, cc) => + implicit val c: CallContext = cc + val rawBody = cc.httpBody.getOrElse("") + for { + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostDeactivateDynamicArtefactJsonV700", 400, Some(cc)) { + if (rawBody.trim.isEmpty) PostDeactivateDynamicArtefactJsonV700(None) + else com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[PostDeactivateDynamicArtefactJsonV700] + } + audit <- Future(MakerChecker.deactivate(targetType, targetId, u.userId, postJson.comment.getOrElse(""))) + .map(unboxFullOrFail(_, Some(cc), s"$DynamicChangeRequestTargetNotFound $targetType $targetId", 404)) + } yield JSONFactory700.createDynamicChangeRequestJsonV700(audit) + } + + private val deactivationDescription = + s"""Deactivate this dynamic artefact immediately. Deactivation reduces capability, so a single holder of `CanApproveDynamicChangeRequest` may do it directly; it is audited as a Dynamic Change Request with operation DEACTIVATE and status APPROVED. Re-activation requires a Dynamic Change Request with operation ACTIVATE approved by a second user. + | + |An inactive artefact is never compiled or executed, regardless of whether maker/checker is enabled. + | + |${userAuthenticationMessage(true)}""".stripMargin + + private val deactivationExample = changeRequestExample.copy(operation = "DEACTIVATE", status = "APPROVED", target_id = "0d1c9e3c-6c2b-4c1e-9a53-2d5b2f0d7f22", request_path = "", proposed_payload = Extraction.decompose(Map("is_active" -> false)), checker_user_id = code.api.util.ExampleValue.userIdExample.value, actioned_at = APIUtil.DateWithMsExampleString) + + val deactivateDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-resource-docs" / dynamicResourceDocId / "deactivation" if dynamicResourceDocId.nonEmpty => + deactivateArtefact(req, DYNAMIC_RESOURCE_DOC, dynamicResourceDocId) + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(deactivateDynamicResourceDoc), + "POST", + "/management/dynamic-resource-docs/DYNAMIC_RESOURCE_DOC_ID/deactivation", + "Deactivate Dynamic Resource Doc", + deactivationDescription, + PostDeactivateDynamicArtefactJsonV700(Some("Suspected data leak; disabling pending review.")), + deactivationExample, + List($AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicChangeRequestTargetNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamicResourceDoc :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(deactivateDynamicResourceDoc) + ) + + val deactivateConnectorMethod: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "connector-methods" / connectorMethodId / "deactivation" if connectorMethodId.nonEmpty => + deactivateArtefact(req, CONNECTOR_METHOD, connectorMethodId) + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(deactivateConnectorMethod), + "POST", + "/management/connector-methods/CONNECTOR_METHOD_ID/deactivation", + "Deactivate Connector Method", + deactivationDescription, + PostDeactivateDynamicArtefactJsonV700(Some("Suspected data leak; disabling pending review.")), + deactivationExample.copy(target_type = "CONNECTOR_METHOD"), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicChangeRequestTargetNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagConnectorMethod :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(deactivateConnectorMethod) + ) + + val deactivateDynamicMessageDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-message-docs" / dynamicMessageDocId / "deactivation" if dynamicMessageDocId.nonEmpty => + deactivateArtefact(req, DYNAMIC_MESSAGE_DOC, dynamicMessageDocId) + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(deactivateDynamicMessageDoc), + "POST", + "/management/dynamic-message-docs/DYNAMIC_MESSAGE_DOC_ID/deactivation", + "Deactivate Dynamic Message Doc", + deactivationDescription, + PostDeactivateDynamicArtefactJsonV700(Some("Suspected data leak; disabling pending review.")), + deactivationExample.copy(target_type = "DYNAMIC_MESSAGE_DOC"), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicChangeRequestTargetNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamicMessageDoc :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(deactivateDynamicMessageDoc) + ) + + val deactivateAbacRule: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "abac-rules" / abacRuleId / "deactivation" if abacRuleId.nonEmpty => + deactivateArtefact(req, ABAC_RULE, abacRuleId) + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(deactivateAbacRule), + "POST", + "/management/abac-rules/ABAC_RULE_ID/deactivation", + "Deactivate ABAC Rule", + deactivationDescription, + PostDeactivateDynamicArtefactJsonV700(Some("Suspected data leak; disabling pending review.")), + deactivationExample.copy(target_type = "ABAC_RULE"), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicChangeRequestTargetNotFound, UnknownError), + apiTagDynamicChangeRequest :: apiTagABAC :: Nil, + Some(List(ApiRole.canApproveDynamicChangeRequest)), + http4sPartialFunction = Some(deactivateAbacRule) + ) + + // All routes combined (without middleware - for direct use). // // Routes are sorted automatically by URL template specificity (segment count, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 18ed109f68..e69f6be691 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -23,6 +23,8 @@ import org.apache.commons.lang3.StringUtils import com.openbankproject.commons.model.{AccountAttribute, AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankAccount, BankId, BankIdAccountId, CoreAccount, TransactionRequest, TransactionRequestCommonBodyJSON, User} import com.openbankproject.commons.util.ApiVersion import java.util.Date +import org.json4s.{Extraction, JValue} +import org.json4s.JsonAST.{JNothing, JString} import net.liftweb.common.Full import net.liftweb.mapper.{Ascending, By, By_<=, Descending, MaxRows, OrderBy} @@ -40,7 +42,10 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { updated_by_user_id: Option[String], method_body_hash: Option[String], created_at: Option[String], - updated_at: Option[String] + updated_at: Option[String], + // maker/checker: the body hash a checker approved (None when never approved) and the active flag + approved_hash: Option[String] = None, + is_active: Option[Boolean] = None ) case class DynamicResourceDocProvenanceJsonV700(dynamic_resource_doc: JsonDynamicResourceDoc, provenance: ProvenanceJsonV700) case class DynamicResourceDocsProvenanceJsonV700(dynamic_resource_docs: List[DynamicResourceDocProvenanceJsonV700]) @@ -57,7 +62,8 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { DynamicResourceDoc.getJsonDynamicResourceDoc(entity), ProvenanceJsonV700( blankToNone(entity.CreatedByUserId.get), blankToNone(entity.UpdatedByUserId.get), - blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get)) + blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get), + blankToNone(entity.ApprovedHash.get), Some(entity.IsActive.get)) ) def createConnectorMethodProvenanceJsonV700(entity: ConnectorMethod): ConnectorMethodProvenanceJsonV700 = @@ -65,7 +71,8 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { ConnectorMethod.getJsonConnectorMethod(entity), ProvenanceJsonV700( blankToNone(entity.CreatedByUserId.get), blankToNone(entity.UpdatedByUserId.get), - blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get)) + blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get), + blankToNone(entity.ApprovedHash.get), Some(entity.IsActive.get)) ) def createDynamicMessageDocProvenanceJsonV700(entity: DynamicMessageDoc): DynamicMessageDocProvenanceJsonV700 = @@ -73,7 +80,83 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { DynamicMessageDoc.getJsonDynamicMessageDoc(entity), ProvenanceJsonV700( blankToNone(entity.CreatedByUserId.get), blankToNone(entity.UpdatedByUserId.get), - blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get)) + blankToNone(entity.MethodBodyHash.get), formatDateOpt(entity.createdAt.get), formatDateOpt(entity.updatedAt.get), + blankToNone(entity.ApprovedHash.get), Some(entity.IsActive.get)) + ) + + // ─── Maker/checker: dynamic change requests (design: MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md) ─── + case class PostDynamicChangeRequestJsonV700( + target_type: String, + operation: String, + target_id: Option[String], + bank_id: Option[String], + proposed_payload: JValue, + business_justification: Option[String] + ) + case class PostApproveDynamicChangeRequestJsonV700(payload_hash: String, checker_comment: Option[String]) + case class PostRejectDynamicChangeRequestJsonV700(comment: String) + case class PostWithdrawDynamicChangeRequestJsonV700(comment: Option[String]) + case class PostDeactivateDynamicArtefactJsonV700(comment: Option[String]) + + case class DynamicChangeRequestJsonV700( + dynamic_change_request_id: String, + target_type: String, + target_id: String, + operation: String, + status: String, + request_verb: String, + request_path: String, + payload_hash: String, + current_payload_hash: String, + proposed_payload: JValue, + current_payload: JValue, + requestor_user_id: String, + business_justification: String, + checker_user_id: String, + checker_comment: String, + created_at: String, + actioned_at: String, + expires_at: String + ) + case class DynamicChangeRequestsJsonV700(dynamic_change_requests: List[DynamicChangeRequestJsonV700]) + + private def parseOrString(s: String): JValue = + com.openbankproject.commons.util.JsonAliases.parseOpt(Option(s).getOrElse("")).getOrElse(if (StringUtils.isBlank(s)) JNothing else JString(s)) + + /** The live target's JSON, so a client can diff proposed vs current; JNothing when it does not exist. */ + def currentPayloadOf(targetType: String, targetId: String): JValue = { + import code.abacrule.MappedAbacRuleProvider + import com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType._ + if (StringUtils.isBlank(targetId)) JNothing + else scala.util.Try(com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType.withName(targetType)).toOption.map { + case DYNAMIC_RESOURCE_DOC => code.dynamicResourceDoc.DynamicResourceDocProvider.provider.vend.getById(None, targetId).map(Extraction.decompose(_)).getOrElse(JNothing) + case DYNAMIC_MESSAGE_DOC => code.dynamicMessageDoc.DynamicMessageDocProvider.provider.vend.getById(None, targetId).map(Extraction.decompose(_)).getOrElse(JNothing) + case CONNECTOR_METHOD => code.connectormethod.ConnectorMethodProvider.provider.vend.getById(targetId).map(Extraction.decompose(_)).getOrElse(JNothing) + case ABAC_RULE => MappedAbacRuleProvider.getAbacRuleById(targetId).map(r => Extraction.decompose(JSONFactory600.createAbacRuleJsonV600(r))).getOrElse(JNothing) + case _ => JNothing + }.getOrElse(JNothing) + } + + def createDynamicChangeRequestJsonV700(r: code.dynamicchangerequest.DynamicChangeRequestTrait): DynamicChangeRequestJsonV700 = + DynamicChangeRequestJsonV700( + dynamic_change_request_id = r.dynamicChangeRequestId, + target_type = r.targetType, + target_id = r.targetId, + operation = r.operation, + status = r.status, + request_verb = r.requestVerb, + request_path = r.requestPath, + payload_hash = r.payloadHash, + current_payload_hash = r.currentPayloadHash, + proposed_payload = parseOrString(r.proposedPayload), + current_payload = currentPayloadOf(r.targetType, r.targetId), + requestor_user_id = r.requestorUserId, + business_justification = r.businessJustification, + checker_user_id = r.checkerUserId, + checker_comment = r.checkerComment, + created_at = APIUtil.formatDate(r.created), + actioned_at = r.actionedAt.map(APIUtil.formatDate).getOrElse(""), + expires_at = r.expiresAt.map(APIUtil.formatDate).getOrElse("") ) case class ErrorMessageEntryJsonV700(code: String, name: String, message: String) diff --git a/obp-api/src/main/scala/code/bankconnectors/DoobieBusinessStatusQueries.scala b/obp-api/src/main/scala/code/bankconnectors/DoobieBusinessStatusQueries.scala index 1dbb97c1d7..609d8ec439 100644 --- a/obp-api/src/main/scala/code/bankconnectors/DoobieBusinessStatusQueries.scala +++ b/obp-api/src/main/scala/code/bankconnectors/DoobieBusinessStatusQueries.scala @@ -32,6 +32,24 @@ object DoobieBusinessStatusQueries { AND status = $guardStatus""".update.run ) + /** DynamicChangeRequest (maker/checker for dynamic code): actioned once, from INITIATED. */ + def conditionalDynamicChangeRequestStatus( + dynamicChangeRequestId: Long, + guardStatus: String, + newStatus: String, + checkerUserId: String, + checkerComment: String + ): Int = DoobieUtil.runUpdate( + sql"""UPDATE DynamicChangeRequest + SET status = $newStatus, + checkeruserid = $checkerUserId, + checkercomment = $checkerComment, + actionedat = NOW(), + updatedat = NOW() + WHERE id = $dynamicChangeRequestId + AND status = $guardStatus""".update.run + ) + /** MappedAccountApplication: transition only from the guard status (a one-shot decision). */ def conditionalAccountApplicationStatus(accountApplicationId: Long, guardStatus: String, newStatus: String): Int = DoobieUtil.runUpdate( diff --git a/obp-api/src/main/scala/code/bankconnectors/DynamicConnector.scala b/obp-api/src/main/scala/code/bankconnectors/DynamicConnector.scala index 75d68d31c2..8ea8ccdd75 100644 --- a/obp-api/src/main/scala/code/bankconnectors/DynamicConnector.scala +++ b/obp-api/src/main/scala/code/bankconnectors/DynamicConnector.scala @@ -36,7 +36,9 @@ object DynamicConnector { } private def getFunction(bankId: Option[String], process: String):Box[DynamicFunction] = { - DynamicMessageDocProvider.provider.vend.getByProcess(bankId, process) map { + // Maker/checker execution guard (active + approved hash); see code.dynamicchangerequest.MakerChecker + DynamicMessageDocProvider.provider.vend.getByProcess(bankId, process) + .filter(_.dynamicMessageDocId.exists(code.dynamicchangerequest.MakerChecker.isExecutableDynamicMessageDoc)) map { case v :JsonDynamicMessageDoc => createFunction(v.programmingLang, v.decodedMethodBody).openOrThrowException(s"InternalConnector method compile fail") } diff --git a/obp-api/src/main/scala/code/bankconnectors/InternalConnector.scala b/obp-api/src/main/scala/code/bankconnectors/InternalConnector.scala index 3d5b51bfd6..7ee8447885 100644 --- a/obp-api/src/main/scala/code/bankconnectors/InternalConnector.scala +++ b/obp-api/src/main/scala/code/bankconnectors/InternalConnector.scala @@ -69,7 +69,9 @@ object InternalConnector { methodNameToSignature.contains(methodName) private def getFunction(methodName: String) = { - ConnectorMethodProvider.provider.vend.getByMethodNameWithCache(methodName) map { + // Maker/checker execution guard (active + approved hash); see code.dynamicchangerequest.MakerChecker + ConnectorMethodProvider.provider.vend.getByMethodNameWithCache(methodName) + .filter(_.connectorMethodId.exists(code.dynamicchangerequest.MakerChecker.isExecutableConnectorMethod)) map { case v :JsonConnectorMethod => createFunction(methodName, v.decodedMethodBody, v.programmingLang).openOrThrowException(s"InternalConnector method compile fail, method name $methodName") } diff --git a/obp-api/src/main/scala/code/connectormethod/ConnectorMethod.scala b/obp-api/src/main/scala/code/connectormethod/ConnectorMethod.scala index 5108c09971..fc37117569 100644 --- a/obp-api/src/main/scala/code/connectormethod/ConnectorMethod.scala +++ b/obp-api/src/main/scala/code/connectormethod/ConnectorMethod.scala @@ -19,6 +19,14 @@ class ConnectorMethod extends LongKeyedMapper[ConnectorMethod] with IdPK with Cr object CreatedByUserId extends MappedString(this, 255) object UpdatedByUserId extends MappedString(this, 255) object MethodBodyHash extends MappedString(this, 64) + // Maker/checker (see MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md): the runtime only loads this row when + // IsActive is true and, when maker/checker is enabled for this target type, when MethodBodyHash + // equals ApprovedHash. ApprovedHash is written only by an approved DynamicChangeRequest (or the + // one-off seeding of pre-existing rows when the feature is first enabled), never from a request body. + object ApprovedHash extends MappedString(this, 64) + object IsActive extends MappedBoolean(this) { + override def defaultValue = true + } } diff --git a/obp-api/src/main/scala/code/dynamicMessageDoc/DynamicMessageDoc.scala b/obp-api/src/main/scala/code/dynamicMessageDoc/DynamicMessageDoc.scala index f22e742c92..9e72fe9b07 100644 --- a/obp-api/src/main/scala/code/dynamicMessageDoc/DynamicMessageDoc.scala +++ b/obp-api/src/main/scala/code/dynamicMessageDoc/DynamicMessageDoc.scala @@ -31,6 +31,14 @@ class DynamicMessageDoc extends LongKeyedMapper[DynamicMessageDoc] with IdPK wit object CreatedByUserId extends MappedString(this, 255) object UpdatedByUserId extends MappedString(this, 255) object MethodBodyHash extends MappedString(this, 64) + // Maker/checker (see MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md): the runtime only loads this row when + // IsActive is true and, when maker/checker is enabled for this target type, when MethodBodyHash + // equals ApprovedHash. ApprovedHash is written only by an approved DynamicChangeRequest (or the + // one-off seeding of pre-existing rows when the feature is first enabled), never from a request body. + object ApprovedHash extends MappedString(this, 64) + object IsActive extends MappedBoolean(this) { + override def defaultValue = true + } } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index a96c164350..f15438ca2a 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -31,6 +31,14 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object CreatedByUserId extends MappedString(this, 255) object UpdatedByUserId extends MappedString(this, 255) object MethodBodyHash extends MappedString(this, 64) + // Maker/checker (see MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md): the runtime only loads this row when + // IsActive is true and, when maker/checker is enabled for this target type, when MethodBodyHash + // equals ApprovedHash. ApprovedHash is written only by an approved DynamicChangeRequest (or the + // one-off seeding of pre-existing rows when the feature is first enabled), never from a request body. + object ApprovedHash extends MappedString(this, 64) + object IsActive extends MappedBoolean(this) { + override def defaultValue = true + } } diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequest.scala b/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequest.scala new file mode 100644 index 0000000000..520db7c38c --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequest.scala @@ -0,0 +1,129 @@ +package code.dynamicchangerequest + +import java.util.Date +import code.api.util.ErrorMessages +import code.util.MappedUUID +import com.openbankproject.commons.model.enums.DynamicChangeRequestStatus +import net.liftweb.common.{Box, Failure, Full} +import net.liftweb.mapper._ +import net.liftweb.util.Helpers.tryo + +object MappedDynamicChangeRequestProvider extends DynamicChangeRequestProvider { + + override def create( + targetType: String, + targetId: String, + operation: String, + requestVerb: String, + requestPath: String, + proposedPayload: String, + payloadHash: String, + currentPayloadHash: String, + requestorUserId: String, + businessJustification: String, + expiresAt: Option[Date] + ): Box[DynamicChangeRequestTrait] = tryo { + DynamicChangeRequest.create + .TargetType(targetType) + .TargetId(targetId) + .Operation(operation) + .RequestVerb(requestVerb) + .RequestPath(requestPath) + .ProposedPayload(proposedPayload) + .PayloadHash(payloadHash) + .CurrentPayloadHash(currentPayloadHash) + .Status(DynamicChangeRequestStatus.INITIATED.toString) + .RequestorUserId(requestorUserId) + .BusinessJustification(businessJustification) + .CheckerUserId("") + .CheckerComment("") + .ExpiresAt(expiresAt.orNull) + .saveMe() + } + + override def getById(dynamicChangeRequestId: String): Box[DynamicChangeRequestTrait] = + DynamicChangeRequest.find(By(DynamicChangeRequest.DynamicChangeRequestId, dynamicChangeRequestId)) + + override def getAll( + status: Option[String], + targetType: Option[String], + targetId: Option[String], + requestorUserId: Option[String] + ): List[DynamicChangeRequestTrait] = { + val filters: List[QueryParam[DynamicChangeRequest]] = + status.map(By(DynamicChangeRequest.Status, _)).toList ::: + targetType.map(By(DynamicChangeRequest.TargetType, _)).toList ::: + targetId.map(By(DynamicChangeRequest.TargetId, _)).toList ::: + requestorUserId.map(By(DynamicChangeRequest.RequestorUserId, _)).toList ::: + List(OrderBy(DynamicChangeRequest.id, Descending)) + DynamicChangeRequest.findAll(filters: _*) + } + + override def getByRequestorUserId(requestorUserId: String): List[DynamicChangeRequestTrait] = + DynamicChangeRequest.findAll( + By(DynamicChangeRequest.RequestorUserId, requestorUserId), + OrderBy(DynamicChangeRequest.id, Descending)) + + override def updateStatus( + dynamicChangeRequestId: String, + status: String, + checkerUserId: String, + checkerComment: String + ): Box[DynamicChangeRequestTrait] = { + getById(dynamicChangeRequestId).flatMap { request => + // Atomic guarded transition: a request is actioned once, from INITIATED. The loser of a + // concurrent approve/reject gets 0 rows -> Failure, instead of silently overwriting the decision. + val rows = code.bankconnectors.DoobieBusinessStatusQueries.conditionalDynamicChangeRequestStatus( + request.asInstanceOf[DynamicChangeRequest].id.get, + DynamicChangeRequestStatus.INITIATED.toString, status, checkerUserId, checkerComment) + if (rows == 1) getById(dynamicChangeRequestId) + else Failure(ErrorMessages.DynamicChangeRequestNotInitiated) + } + } +} + +class DynamicChangeRequest extends DynamicChangeRequestTrait with LongKeyedMapper[DynamicChangeRequest] with IdPK with CreatedUpdated { + + def getSingleton = DynamicChangeRequest + + object DynamicChangeRequestId extends MappedUUID(this) + object TargetType extends MappedString(this, 64) + object TargetId extends MappedString(this, 255) + object Operation extends MappedString(this, 32) + object RequestVerb extends MappedString(this, 16) + object RequestPath extends MappedString(this, 1024) + object ProposedPayload extends MappedText(this) + object PayloadHash extends MappedString(this, 64) + object CurrentPayloadHash extends MappedString(this, 64) + object Status extends MappedString(this, 32) + object RequestorUserId extends MappedString(this, 255) + object BusinessJustification extends MappedText(this) + object CheckerUserId extends MappedString(this, 255) + object CheckerComment extends MappedText(this) + object ActionedAt extends MappedDateTime(this) + object ExpiresAt extends MappedDateTime(this) + + override def dynamicChangeRequestId: String = DynamicChangeRequestId.get + override def targetType: String = TargetType.get + override def targetId: String = Option(TargetId.get).getOrElse("") + override def operation: String = Operation.get + override def requestVerb: String = Option(RequestVerb.get).getOrElse("") + override def requestPath: String = Option(RequestPath.get).getOrElse("") + override def proposedPayload: String = Option(ProposedPayload.get).getOrElse("") + override def payloadHash: String = Option(PayloadHash.get).getOrElse("") + override def currentPayloadHash: String = Option(CurrentPayloadHash.get).getOrElse("") + override def status: String = Status.get + override def requestorUserId: String = RequestorUserId.get + override def businessJustification: String = Option(BusinessJustification.get).getOrElse("") + override def checkerUserId: String = Option(CheckerUserId.get).getOrElse("") + override def checkerComment: String = Option(CheckerComment.get).getOrElse("") + override def created: Date = createdAt.get + override def updated: Date = updatedAt.get + override def actionedAt: Option[Date] = Option(ActionedAt.get) + override def expiresAt: Option[Date] = Option(ExpiresAt.get) +} + +object DynamicChangeRequest extends DynamicChangeRequest with LongKeyedMetaMapper[DynamicChangeRequest] { + override def dbTableName = "DynamicChangeRequest" + override def dbIndexes = UniqueIndex(DynamicChangeRequestId) :: Index(TargetType, TargetId) :: Index(RequestorUserId) :: Index(Status) :: super.dbIndexes +} diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequestTrait.scala b/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequestTrait.scala new file mode 100644 index 0000000000..d251ca253f --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicchangerequest/DynamicChangeRequestTrait.scala @@ -0,0 +1,77 @@ +package code.dynamicchangerequest + +import java.util.Date +import net.liftweb.common.Box +import net.liftweb.util.SimpleInjector + +object DynamicChangeRequestTrait extends SimpleInjector { + val dynamicChangeRequest = new Inject(() => buildOne) {} + + def buildOne: DynamicChangeRequestProvider = MappedDynamicChangeRequestProvider +} + +/** + * One row per maker/checker request against a runtime-supplied artefact (dynamic resource doc, + * connector method, dynamic message doc, ABAC rule ...). The row is never deleted: the table is + * the audit log. There is deliberately no bank column: dynamic code runs in the shared JVM and + * approval is system level; the intercepted request path records the scope the maker used. + */ +trait DynamicChangeRequestTrait { + def dynamicChangeRequestId: String + def targetType: String + def targetId: String + def operation: String + /** the original call the maker made, e.g. POST /obp/v4.0.0/management/banks/BANK_ID/dynamic-resource-docs */ + def requestVerb: String + def requestPath: String + /** the exact JSON body the maker sent, stored verbatim */ + def proposedPayload: String + /** SHA-256 of the canonicalised proposed payload; what the checker approves */ + def payloadHash: String + /** body hash of the live target at submission time; empty for CREATE */ + def currentPayloadHash: String + def status: String + def requestorUserId: String + def businessJustification: String + def checkerUserId: String + def checkerComment: String + def created: Date + def updated: Date + def actionedAt: Option[Date] + def expiresAt: Option[Date] +} + +trait DynamicChangeRequestProvider { + def create( + targetType: String, + targetId: String, + operation: String, + requestVerb: String, + requestPath: String, + proposedPayload: String, + payloadHash: String, + currentPayloadHash: String, + requestorUserId: String, + businessJustification: String, + expiresAt: Option[Date] + ): Box[DynamicChangeRequestTrait] + + def getById(dynamicChangeRequestId: String): Box[DynamicChangeRequestTrait] + + def getAll( + status: Option[String], + targetType: Option[String], + targetId: Option[String], + requestorUserId: Option[String] + ): List[DynamicChangeRequestTrait] + + def getByRequestorUserId(requestorUserId: String): List[DynamicChangeRequestTrait] + + /** Guarded INITIATED -> status transition; Failure when the row was already actioned. */ + def updateStatus( + dynamicChangeRequestId: String, + status: String, + checkerUserId: String, + checkerComment: String + ): Box[DynamicChangeRequestTrait] +} diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala new file mode 100644 index 0000000000..4d7702e2c9 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala @@ -0,0 +1,521 @@ +package code.dynamicchangerequest + +import java.net.URLDecoder +import java.util.Date + +import code.abacrule.{AbacRule, AbacRuleEngine, MappedAbacRuleProvider} +import code.api.Constant +import code.api.dynamic.endpoint.helper.CompiledObjects +import code.api.util.APIUtil.{getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} +import code.api.util.DynamicUtil.Validation +import code.api.util.{CallContext, ErrorMessages} +import code.api.v6_0_0.{CreateAbacRuleJsonV600, UpdateAbacRuleJsonV600} +import code.bankconnectors.{DynamicConnector, InternalConnector} +import code.connectormethod.{ConnectorMethod, ConnectorMethodProvider, JsonConnectorMethod, JsonConnectorMethodMethodBody} +import code.dynamicMessageDoc.{DynamicMessageDoc, DynamicMessageDocProvider, JsonDynamicMessageDoc} +import code.dynamicResourceDoc.{DynamicResourceDoc, DynamicResourceDocProvider, JsonDynamicResourceDoc} +import code.util.Helper.MdcLoggable +import com.openbankproject.commons.model.enums.DynamicChangeRequestOperation._ +import com.openbankproject.commons.model.enums.DynamicChangeRequestStatus +import com.openbankproject.commons.model.enums.DynamicChangeRequestTargetType._ +import com.openbankproject.commons.model.enums.{DynamicChangeRequestOperation, DynamicChangeRequestTargetType} +import com.openbankproject.commons.util.JsonAliases.{compactRender, parse} +import net.liftweb.common.{Box, Empty, Failure, Full} +import org.json4s.Formats +import net.liftweb.mapper.By +import net.liftweb.util.Helpers.tryo +import org.apache.commons.lang3.StringUtils +import org.json4s.JsonAST.{JArray, JNothing, JObject, JValue} + +/** + * Maker/checker for runtime-supplied code and configuration. Design: MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md. + * + * Three responsibilities: + * - configuration + hashing (what is managed, what "the same content" means); + * - the execution guard: the runtime loads a row only when it is active and, for a managed type, + * when its body hash equals the hash a checker approved (enforced at the load site, so it holds + * against direct database edits and stale compile memos); + * - the request lifecycle: intercept a v4/v6 write into the queue, approve (re-validate + apply + * through the same providers the v4/v6 handlers use), reject, withdraw, expire, deactivate. + */ +object MakerChecker extends MdcLoggable { + + implicit private val formats: Formats = code.api.util.CustomJsonFormats.formats + + // ─── configuration ───────────────────────────────────────────────────────── + + /** dynamic_code_requires_approval: when true, writes to the managed target types are queued for a second + * user's approval and the runtime executes only code whose body hash a checker approved. */ + def enabled: Boolean = getPropsAsBoolValue("dynamic_code_requires_approval", false) + + private val defaultTargetTypes = List(DYNAMIC_RESOURCE_DOC, DYNAMIC_MESSAGE_DOC, CONNECTOR_METHOD, ABAC_RULE).map(_.toString) + + def managedTargetTypes: Set[String] = + getPropsValue("dynamic_code_approval_target_types", defaultTargetTypes.mkString(",")) + .split(",").map(_.trim).filter(_.nonEmpty).toSet + + def isManaged(targetType: DynamicChangeRequestTargetType): Boolean = enabled && managedTargetTypes.contains(targetType.toString) + + def requireApprovalForDelete: Boolean = getPropsAsBoolValue("dynamic_code_delete_requires_approval", true) + + def requestTtlHours: Int = getPropsAsIntValue("dynamic_code_approval_request_ttl_hours", 168) + + /** Only these types can be applied by this phase; the enum lists the later phases too. */ + val applicableTargetTypes: Set[DynamicChangeRequestTargetType] = Set(DYNAMIC_RESOURCE_DOC, DYNAMIC_MESSAGE_DOC, CONNECTOR_METHOD, ABAC_RULE) + + // ─── hashing ─────────────────────────────────────────────────────────────── + + /** Sorted keys, no insignificant whitespace, so a client that reorders fields hashes the same. */ + def canonicalJson(jv: JValue): JValue = jv match { + case JObject(fields) => JObject(fields.sortBy(_._1).map { case (k, v) => (k, canonicalJson(v)) }) + case JArray(items) => JArray(items.map(canonicalJson)) + case other => other + } + + /** SHA-256 hex of the canonical form of a JSON payload (falls back to the raw text when it is not JSON). */ + def payloadHash(rawPayload: String): String = { + val raw = Option(rawPayload).getOrElse("") + // A DELETE has no body: parse("") yields JNothing, which cannot be rendered, so hash the raw text. + tryo(parse(raw)).filter(_ != JNothing).map(jv => sha256Hex(compactRender(canonicalJson(jv)))).getOrElse(sha256Hex(raw)) + } + + /** Accept "sha256:" as well as bare hex from clients. */ + def normaliseHash(h: String): String = Option(h).map(_.trim).map(_.stripPrefix("sha256:")).getOrElse("") + + private def blank(s: String): Boolean = StringUtils.isBlank(s) + + private def bodyHashOf(storedHash: String, encodedBody: String): String = + if (!blank(storedHash)) storedHash + else sha256Hex(URLDecoder.decode(Option(encodedBody).getOrElse(""), "UTF-8")) + + /** The live target's body hash, Empty when the target does not exist. */ + def currentBodyHash(targetType: DynamicChangeRequestTargetType, targetId: String): Box[String] = targetType match { + case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) + case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) + case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) + case ABAC_RULE => AbacRule.find(By(AbacRule.AbacRuleId, targetId)).map(r => sha256Hex(Option(r.RuleCode.get).getOrElse(""))) + case _ => Empty + } + + // ─── execution guard ─────────────────────────────────────────────────────── + + private def executable(isActive: Boolean, bodyHash: String, approvedHash: String, targetType: DynamicChangeRequestTargetType): Boolean = + isActive && (!isManaged(targetType) || (!blank(approvedHash) && bodyHash == approvedHash)) + + // Connector methods and message docs are looked up per API call, so the per-row guard is memoised + // briefly (0 in test mode, like the providers' own caches). Approval / deactivation therefore take + // up to this long to propagate to those two families; the resource doc group is already TTL-cached. + private val guardTtl: Int = if (net.liftweb.util.Props.testMode) 0 else getPropsAsIntValue("dynamic_code_approval_guard_cache_ttl_seconds", 10) + private def memoGuard(key: String)(check: => Boolean): Boolean = + code.api.cache.Caching.memoizeSyncWithImMemory(Some(("maker_checker_guard_" + key).intern()))(scala.concurrent.duration.Duration(guardTtl, "seconds"))(check) + + def isExecutableDynamicResourceDoc(dynamicResourceDocId: String): Boolean = + DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, dynamicResourceDocId)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_RESOURCE_DOC)) + .getOrElse(false) + + def isExecutableDynamicMessageDoc(dynamicMessageDocId: String): Boolean = memoGuard("dmd_" + dynamicMessageDocId) { + DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, dynamicMessageDocId)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_MESSAGE_DOC)) + .getOrElse(false) + } + + def isExecutableConnectorMethod(connectorMethodId: String): Boolean = memoGuard("cm_" + connectorMethodId) { + ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, connectorMethodId)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, CONNECTOR_METHOD)) + .getOrElse(false) + } + + /** AbacRuleEngine already checks IsActive; this adds the approved-hash check for managed instances. */ + def isApprovedAbacRule(abacRuleId: String): Boolean = + if (!isManaged(ABAC_RULE)) true + else AbacRule.find(By(AbacRule.AbacRuleId, abacRuleId)) + .map(r => !blank(r.ApprovedHash.get) && sha256Hex(Option(r.RuleCode.get).getOrElse("")) == r.ApprovedHash.get) + .getOrElse(false) + + // ─── submission ──────────────────────────────────────────────────────────── + + private def provider = DynamicChangeRequestTrait.dynamicChangeRequest.vend + + private def expiry(): Option[Date] = { + val hours = requestTtlHours + if (hours <= 0) None else Some(new Date(System.currentTimeMillis() + hours.toLong * 3600L * 1000L)) + } + + /** + * Called by a v4/v6 create/update/delete handler AFTER it has parsed, validated and compiled the + * body exactly as it does today. Some(request) means the write was queued and the handler must + * answer 202 with it; None means maker/checker does not apply and the handler proceeds as before. + */ + def intercept( + targetType: DynamicChangeRequestTargetType, + operation: DynamicChangeRequestOperation, + targetId: Option[String], + cc: CallContext + ): Box[Option[DynamicChangeRequestTrait]] = { + val deleteBypass = operation == DELETE && !requireApprovalForDelete + if (!isManaged(targetType) || deleteBypass) Full(None) + else { + val requestor = cc.user.map(_.userId).openOr("") + if (blank(requestor)) Failure(ErrorMessages.AuthenticatedUserIsRequired) + else submit(targetType, operation, targetId, cc.verb, cc.url, cc.httpBody.getOrElse(""), requestor, "").map(Some(_)) + } + } + + def submit( + targetType: DynamicChangeRequestTargetType, + operation: DynamicChangeRequestOperation, + targetId: Option[String], + requestVerb: String, + requestPath: String, + proposedPayload: String, + requestorUserId: String, + businessJustification: String + ): Box[DynamicChangeRequestTrait] = { + val id = targetId.getOrElse("") + val current: Box[String] = if (operation == CREATE) Full("") else currentBodyHash(targetType, id) + current match { + case Full(currentHash) => + provider.create( + targetType = targetType.toString, + targetId = id, + operation = operation.toString, + requestVerb = requestVerb, + requestPath = requestPath, + proposedPayload = proposedPayload, + payloadHash = payloadHash(proposedPayload), + currentPayloadHash = currentHash, + requestorUserId = requestorUserId, + businessJustification = businessJustification, + expiresAt = expiry() + ) + case _ => Failure(s"${ErrorMessages.DynamicChangeRequestTargetNotFound} ${targetType} $id") + } + } + + // ─── lifecycle ───────────────────────────────────────────────────────────── + + /** Lazily move an overdue INITIATED request to EXPIRED and return the refreshed row. */ + def expireIfDue(request: DynamicChangeRequestTrait): DynamicChangeRequestTrait = + if (request.status == DynamicChangeRequestStatus.INITIATED.toString && request.expiresAt.exists(_.before(new Date()))) + provider.updateStatus(request.dynamicChangeRequestId, DynamicChangeRequestStatus.EXPIRED.toString, "", "expired").openOr(request) + else request + + def approve(request: DynamicChangeRequestTrait, checkerUserId: String, payloadHashFromChecker: String, comment: String): Box[DynamicChangeRequestTrait] = { + val targetType = tryo(DynamicChangeRequestTargetType.withName(request.targetType)).toOption + val operation = tryo(DynamicChangeRequestOperation.withName(request.operation)).toOption + for { + _ <- boolBox(request.status == DynamicChangeRequestStatus.INITIATED.toString, ErrorMessages.DynamicChangeRequestNotInitiated) + _ <- boolBox(checkerUserId != request.requestorUserId, ErrorMessages.MakerCheckerSameUser) + _ <- boolBox(normaliseHash(payloadHashFromChecker) == request.payloadHash, ErrorMessages.DynamicChangeRequestHashMismatch) + tt <- Box(targetType) ?~! s"${ErrorMessages.DynamicChangeRequestTargetTypeNotManaged} ${request.targetType}" + op <- Box(operation) ?~! s"${ErrorMessages.InvalidJsonFormat} operation ${request.operation}" + _ <- boolBox(applicableTargetTypes.contains(tt), s"${ErrorMessages.DynamicChangeRequestTargetTypeNotManaged} ${request.targetType}") + _ <- if (op == CREATE) Full(()) else currentBodyHash(tt, request.targetId) match { + case Full(h) if h == request.currentPayloadHash => Full(()) + case Full(_) => Failure(ErrorMessages.DynamicChangeRequestStale) + case _ => Failure(s"${ErrorMessages.DynamicChangeRequestTargetNotFound} ${request.targetType} ${request.targetId}") + } + // Win the INITIATED -> APPROVED transition BEFORE applying, so a concurrent approve/reject + // cannot apply twice; if apply then fails the row is moved to FAILED with the reason. + approved <- provider.updateStatus(request.dynamicChangeRequestId, DynamicChangeRequestStatus.APPROVED.toString, checkerUserId, comment) + result <- applyRequest(approved, tt, op) match { + case Full(appliedTargetId) => + // A CREATE only knows its target after apply: record it so the request points at the row it made. + if (blank(approved.targetId) && !blank(appliedTargetId)) setTargetId(approved, appliedTargetId) + Full(approved) + case fail: Failure => + markFailed(approved, s"${Option(comment).getOrElse("")} | apply failed: ${fail.messageChain}".trim) + Failure(s"${ErrorMessages.DynamicChangeRequestApplyFailed} ${fail.messageChain}") + case _ => + markFailed(approved, s"${Option(comment).getOrElse("")} | apply failed".trim) + Failure(ErrorMessages.DynamicChangeRequestApplyFailed) + } + refreshed <- provider.getById(result.dynamicChangeRequestId) + } yield refreshed + } + + def reject(request: DynamicChangeRequestTrait, checkerUserId: String, comment: String): Box[DynamicChangeRequestTrait] = + for { + _ <- boolBox(request.status == DynamicChangeRequestStatus.INITIATED.toString, ErrorMessages.DynamicChangeRequestNotInitiated) + _ <- boolBox(checkerUserId != request.requestorUserId, ErrorMessages.MakerCheckerSameUser) + updated <- provider.updateStatus(request.dynamicChangeRequestId, DynamicChangeRequestStatus.REJECTED.toString, checkerUserId, comment) + } yield updated + + def withdraw(request: DynamicChangeRequestTrait, requestorUserId: String, comment: String): Box[DynamicChangeRequestTrait] = + for { + _ <- boolBox(request.status == DynamicChangeRequestStatus.INITIATED.toString, ErrorMessages.DynamicChangeRequestNotInitiated) + _ <- boolBox(requestorUserId == request.requestorUserId, ErrorMessages.DynamicChangeRequestNotRequestor) + updated <- provider.updateStatus(request.dynamicChangeRequestId, DynamicChangeRequestStatus.WITHDRAWN.toString, requestorUserId, comment) + } yield updated + + /** + * Four eyes to enable, one pair to disable: a single approver deactivates directly. Audited as a + * DEACTIVATE row written straight through to APPROVED with requestor = checker. + */ + def deactivate(targetType: DynamicChangeRequestTargetType, targetId: String, checkerUserId: String, comment: String): Box[DynamicChangeRequestTrait] = + for { + _ <- boolBox(applicableTargetTypes.contains(targetType), s"${ErrorMessages.DynamicChangeRequestTargetTypeNotManaged} $targetType") + currentHash <- currentBodyHash(targetType, targetId) ?~! s"${ErrorMessages.DynamicChangeRequestTargetNotFound} $targetType $targetId" + _ <- setActive(targetType, targetId, active = false) + audit <- provider.create(targetType.toString, targetId, DEACTIVATE.toString, "POST", "", """{"is_active":false}""", + payloadHash("""{"is_active":false}"""), currentHash, checkerUserId, comment, None) + done <- provider.updateStatus(audit.dynamicChangeRequestId, DynamicChangeRequestStatus.APPROVED.toString, checkerUserId, comment) + } yield done + + private def setTargetId(request: DynamicChangeRequestTrait, targetId: String): Unit = + DynamicChangeRequest.find(By(DynamicChangeRequest.DynamicChangeRequestId, request.dynamicChangeRequestId)).foreach { row => + row.TargetId(targetId).save + } + + private def markFailed(request: DynamicChangeRequestTrait, comment: String): Unit = + DynamicChangeRequest.find(By(DynamicChangeRequest.DynamicChangeRequestId, request.dynamicChangeRequestId)).foreach { row => + row.Status(DynamicChangeRequestStatus.FAILED.toString).CheckerComment(comment.take(4000)).save + } + + private def boolBox(condition: Boolean, failMsg: => String): Box[Unit] = if (condition) Full(()) else Failure(failMsg) + + // ─── apply ───────────────────────────────────────────────────────────────── + + private val bankInPath = """.*/banks/([^/]+)/.*""".r + + /** The maker's original call encodes the scope: /management/banks/BANK_ID/... vs /management/... */ + def bankIdFromPath(requestPath: String): Option[String] = requestPath match { + case bankInPath(bankId) if !blank(bankId) => Some(bankId) + case _ => None + } + + /** Applies the request and returns the id of the target it acted on (the new id for a CREATE). */ + private def applyRequest(request: DynamicChangeRequestTrait, targetType: DynamicChangeRequestTargetType, operation: DynamicChangeRequestOperation): Box[String] = { + val result: Box[String] = tryo { + targetType match { + case DYNAMIC_RESOURCE_DOC => applyDynamicResourceDoc(request, operation) + case DYNAMIC_MESSAGE_DOC => applyDynamicMessageDoc(request, operation) + case CONNECTOR_METHOD => applyConnectorMethod(request, operation) + case ABAC_RULE => applyAbacRule(request, operation) + case other => Failure(s"${ErrorMessages.DynamicChangeRequestTargetTypeNotManaged} $other") + } + }.flatMap(identity) + result.foreach(id => invalidateCaches(targetType, id)) + result + } + + private def invalidateCaches(targetType: DynamicChangeRequestTargetType, targetId: String): Unit = targetType match { + case DYNAMIC_RESOURCE_DOC => + Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) + Constant.incrementCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) + case ABAC_RULE => AbacRuleEngine.clearRuleFromCache(targetId) + case _ => () + } + + private def parseAs[T: Manifest](payload: String): Box[T] = + tryo(parse(payload).extract[T]) ?~! s"${ErrorMessages.InvalidJsonFormat} The stored payload is not a ${manifest[T].runtimeClass.getSimpleName}" + + private def compileBox(description: String)(block: => Box[_]): Box[Unit] = { + val compiled: Box[Any] = tryo(block) match { + case Full(inner) => inner + case f: Failure => f + case _ => Empty + } + compiled match { + case Full(_) => Full(()) + case f: Failure => Failure(s"${ErrorMessages.DynamicCodeCompileFail} $description: ${f.messageChain}") + case _ => Failure(s"${ErrorMessages.DynamicCodeCompileFail} $description") + } + } + + private def applyDynamicResourceDoc(request: DynamicChangeRequestTrait, operation: DynamicChangeRequestOperation): Box[String] = { + val bankId = bankIdFromPath(request.requestPath) + val p = DynamicResourceDocProvider.provider.vend + operation match { + case CREATE | UPDATE => + for { + body <- parseAs[JsonDynamicResourceDoc](request.proposedPayload) + _ <- compileBox("dynamic resource doc") { + val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody) + compiled.validateDependency() + Full(compiled) + } + saved <- if (operation == CREATE) { + for { + _ <- boolBox(p.getByVerbAndUrl(bankId, body.requestVerb, body.requestUrl).isEmpty, + s"${ErrorMessages.DynamicResourceDocAlreadyExists} ${body.requestVerb} ${body.requestUrl}") + created <- p.create(bankId, body, Some(request.requestorUserId)) + } yield created + } else p.update(bankId, body.copy(dynamicResourceDocId = Some(request.targetId)), Some(request.requestorUserId)) + id = saved.dynamicResourceDocId.getOrElse("") + _ <- markApproved(DYNAMIC_RESOURCE_DOC, id) + } yield id + case DELETE => p.deleteById(bankId, request.targetId).map(_ => request.targetId) + case ACTIVATE => markApproved(DYNAMIC_RESOURCE_DOC, request.targetId).map(_ => request.targetId) + case other => Failure(s"${ErrorMessages.InvalidJsonFormat} operation $other is not a request operation") + } + } + + private def applyDynamicMessageDoc(request: DynamicChangeRequestTrait, operation: DynamicChangeRequestOperation): Box[String] = { + val bankId = bankIdFromPath(request.requestPath) + val p = DynamicMessageDocProvider.provider.vend + operation match { + case CREATE | UPDATE => + for { + body <- parseAs[JsonDynamicMessageDoc](request.proposedPayload) + _ <- compileBox("dynamic message doc") { + val fn = DynamicConnector.createFunction(body.programmingLang, body.decodedMethodBody) + fn.foreach(Validation.validateDependency(_)) + fn + } + saved <- if (operation == CREATE) { + for { + _ <- boolBox(p.getByProcess(bankId, body.process).isEmpty, s"${ErrorMessages.DynamicMessageDocAlreadyExists} ${body.process}") + created <- p.create(bankId, body, Some(request.requestorUserId)) + } yield created + } else p.update(bankId, body.copy(dynamicMessageDocId = Some(request.targetId)), Some(request.requestorUserId)) + id = saved.dynamicMessageDocId.getOrElse("") + _ <- markApproved(DYNAMIC_MESSAGE_DOC, id) + } yield id + case DELETE => p.deleteById(bankId, request.targetId).map(_ => request.targetId) + case ACTIVATE => markApproved(DYNAMIC_MESSAGE_DOC, request.targetId).map(_ => request.targetId) + case other => Failure(s"${ErrorMessages.InvalidJsonFormat} operation $other is not a request operation") + } + } + + private def applyConnectorMethod(request: DynamicChangeRequestTrait, operation: DynamicChangeRequestOperation): Box[String] = { + val p = ConnectorMethodProvider.provider.vend + operation match { + case CREATE => + for { + body <- parseAs[JsonConnectorMethod](request.proposedPayload) + _ <- boolBox(p.getByMethodNameWithoutCache(body.methodName).isEmpty, s"${ErrorMessages.ConnectorMethodAlreadyExists} ${body.methodName}") + _ <- compileBox("connector method") { + val fn = InternalConnector.createFunction(body.methodName, body.decodedMethodBody, body.programmingLang) + fn.foreach(Validation.validateDependency(_)) + fn + } + created <- p.create(body, Some(request.requestorUserId)) + id = created.connectorMethodId.getOrElse("") + _ <- markApproved(CONNECTOR_METHOD, id) + } yield id + case UPDATE => + for { + body <- parseAs[JsonConnectorMethodMethodBody](request.proposedPayload) + existing <- p.getById(request.targetId) ?~! s"${ErrorMessages.ConnectorMethodNotFound} ${request.targetId}" + _ <- compileBox("connector method") { + val fn = InternalConnector.createFunction(existing.methodName, body.decodedMethodBody, body.programmingLang) + fn.foreach(Validation.validateDependency(_)) + fn + } + _ <- p.update(request.targetId, body.methodBody, body.programmingLang, Some(request.requestorUserId)) + _ <- markApproved(CONNECTOR_METHOD, request.targetId) + } yield request.targetId + case DELETE => p.deleteById(request.targetId).map(_ => request.targetId) + case ACTIVATE => markApproved(CONNECTOR_METHOD, request.targetId).map(_ => request.targetId) + case other => Failure(s"${ErrorMessages.InvalidJsonFormat} operation $other is not a request operation") + } + } + + private def applyAbacRule(request: DynamicChangeRequestTrait, operation: DynamicChangeRequestOperation): Box[String] = operation match { + case CREATE => + for { + body <- parseAs[CreateAbacRuleJsonV600](request.proposedPayload) + _ <- AbacRuleEngine.validateRuleCode(body.rule_code) + rule <- MappedAbacRuleProvider.createAbacRule(body.rule_name, body.rule_code, body.description, body.policy, body.is_active, request.requestorUserId) + _ <- markApproved(ABAC_RULE, rule.abacRuleId) + } yield rule.abacRuleId + case UPDATE => + for { + body <- parseAs[UpdateAbacRuleJsonV600](request.proposedPayload) + _ <- AbacRuleEngine.validateRuleCode(body.rule_code) + _ <- MappedAbacRuleProvider.updateAbacRule(request.targetId, body.rule_name, body.rule_code, body.description, body.policy, body.is_active, request.requestorUserId) + _ <- markApproved(ABAC_RULE, request.targetId) + } yield request.targetId + case DELETE => MappedAbacRuleProvider.deleteAbacRule(request.targetId).map(_ => request.targetId) + case ACTIVATE => markApproved(ABAC_RULE, request.targetId).map(_ => request.targetId) + case other => Failure(s"${ErrorMessages.InvalidJsonFormat} operation $other is not a request operation") + } + + /** Record that the row's current body is the approved one, and make it active. */ + private def markApproved(targetType: DynamicChangeRequestTargetType, targetId: String): Box[Unit] = tryo { + targetType match { + case DYNAMIC_RESOURCE_DOC => + DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map { r => + val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () + } + case DYNAMIC_MESSAGE_DOC => + DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map { r => + val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () + } + case CONNECTOR_METHOD => + ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map { r => + val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () + } + case ABAC_RULE => + AbacRule.find(By(AbacRule.AbacRuleId, targetId)).map { r => + r.ApprovedHash(sha256Hex(Option(r.RuleCode.get).getOrElse(""))).IsActive(true).save; () + } + case _ => Empty + } + }.flatMap(b => b ?~! s"${ErrorMessages.DynamicChangeRequestTargetNotFound} $targetType $targetId") + + private def setActive(targetType: DynamicChangeRequestTargetType, targetId: String, active: Boolean): Box[Unit] = { + val done: Box[Unit] = targetType match { + case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => { r.IsActive(active).save; () }) + case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => { r.IsActive(active).save; () }) + case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => { r.IsActive(active).save; () }) + case ABAC_RULE => AbacRule.find(By(AbacRule.AbacRuleId, targetId)).map(r => { r.IsActive(active).save; () }) + case _ => Empty + } + done.foreach(_ => invalidateCaches(targetType, targetId)) + done ?~! s"${ErrorMessages.DynamicChangeRequestTargetNotFound} $targetType $targetId" + } + + // ─── boot ────────────────────────────────────────────────────────────────── + + /** MigrationScriptLog entry that records the one-off seeding below; the seed never runs twice on a database. */ + val seedMigrationName = "seedDynamicCodeApprovedHashes" + + /** + * Run ONCE per database, the first time the instance boots with dynamic_code_requires_approval=true. + * Rows that predate the feature have no ApprovedHash and would stop executing, so their current body + * is recorded as approved and the run is logged in MigrationScriptLog under `seedMigrationName`. + * It is deliberately not repeated at later boots: after the seed, the only way a row gains an + * ApprovedHash is a checker's approval, so a row written straight into the database (or created while + * the feature was switched off) stays unexecutable until a second person ACTIVATEs it through a change + * request. A pending UPDATE never modifies the live row, so it is unaffected by the seed. + */ + def seedApprovedHashesIfEnabled(): Unit = if (enabled) { + val logProvider = code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend + if (logProvider.isExecuted(seedMigrationName)) { + logger.info(s"dynamic_code_requires_approval: ApprovedHash seeding ($seedMigrationName) already ran on this database; rows without an approved hash will not execute until approved") + } else { + val start = System.currentTimeMillis() + def seed[T](name: String, rows: List[T])(hashOf: T => String, write: (T, String) => Unit): String = { + rows.foreach(r => write(r, hashOf(r))) + s"$name: ${rows.size}" + } + tryo { + List( + seed("DynamicResourceDoc", DynamicResourceDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( + r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + seed("DynamicMessageDoc", DynamicMessageDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( + r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + seed("ConnectorMethod", ConnectorMethod.findAll().filter(r => blank(r.ApprovedHash.get)))( + r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + seed("AbacRule", AbacRule.findAll().filter(r => blank(r.ApprovedHash.get)))( + r => sha256Hex(Option(r.RuleCode.get).getOrElse("")), (r, h) => { r.ApprovedHash(h).save; () }) + ).mkString(", ") + } match { + case Full(summary) => + val comment = s"Seeded ApprovedHash from the current body on pre-existing rows ($summary); their current code is treated as approved" + logger.warn(s"dynamic_code_requires_approval: $comment") + logProvider.saveLog(seedMigrationName, code.api.util.APIUtil.gitCommit, true, start, System.currentTimeMillis(), comment) + case f: Failure => + logger.error(s"dynamic_code_requires_approval: seeding ApprovedHash failed, will retry at next boot: ${f.messageChain}") + logProvider.saveLog(seedMigrationName, code.api.util.APIUtil.gitCommit, false, start, System.currentTimeMillis(), f.messageChain) + case _ => () + } + } + } +} diff --git a/obp-api/src/main/scala/code/obp/grpc/ObpGrpcServer.scala b/obp-api/src/main/scala/code/obp/grpc/ObpGrpcServer.scala index e5bf4aa98a..7a4482772a 100644 --- a/obp-api/src/main/scala/code/obp/grpc/ObpGrpcServer.scala +++ b/obp-api/src/main/scala/code/obp/grpc/ObpGrpcServer.scala @@ -25,7 +25,8 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.control.NonFatal /** - * OBP gRPC server — serves banking RPCs (ObpService) and chat streaming RPCs (ChatStreamService). + * OBP gRPC server — serves banking RPCs (ObpService), chat streaming RPCs (ChatStreamService) + * and signal channel RPCs (SignalChannelsService: publish/fetch/list plus a live Subscribe stream). * Enable via grpc.server.enabled=true in props. */ object ObpGrpcServer { @@ -57,6 +58,7 @@ class ObpGrpcServer(executionContext: ExecutionContext, port: Int = ObpGrpcServe @volatile private[this] var startedChatBus = false @volatile private[this] var startedLogCacheBus = false @volatile private[this] var startedMetricsBus = false + @volatile private[this] var startedSignalBus = false @volatile private[this] var shutdownHook: scala.sys.ShutdownHookThread = null def start(): Unit = { @@ -97,6 +99,11 @@ class ObpGrpcServer(executionContext: ExecutionContext, port: Int = ObpGrpcServe code.logcache.LogCacheEventBus.start() startedLogCacheBus = !logCacheWasRunning && code.logcache.LogCacheEventBus.isRunning + // Start signal event bus for the SignalChannelsService Subscribe stream + val signalWasRunning = code.signal.SignalEventBus.isRunning + code.signal.SignalEventBus.start() + startedSignalBus = !signalWasRunning && code.signal.SignalEventBus.isRunning + // Start metrics event bus (no-op if grpc.metrics_stream.enabled=false) val metricsWasRunning = code.metricsstream.MetricsEventBus.isRunning code.metricsstream.MetricsEventBus.start() @@ -106,6 +113,8 @@ class ObpGrpcServer(executionContext: ExecutionContext, port: Int = ObpGrpcServe .addService(ObpServiceGrpc.bindService(ObpServiceImpl, executionContext)) .addService(code.obp.grpc.chat.api.ChatStreamServiceGrpc.bindService( code.obp.grpc.chat.ChatStreamServiceImpl, executionContext)) + .addService(code.obp.grpc.signal.api.SignalChannelsServiceGrpc.bindService( + code.obp.grpc.signal.SignalChannelsServiceImpl, executionContext)) .addService(io.grpc.protobuf.services.ProtoReflectionService.newInstance()) .intercept(new code.obp.grpc.chat.AuthInterceptor()) @@ -140,6 +149,7 @@ class ObpGrpcServer(executionContext: ExecutionContext, port: Int = ObpGrpcServe if (startedChatBus) { code.chat.ChatEventBus.stop(); startedChatBus = false } if (startedLogCacheBus) { code.logcache.LogCacheEventBus.stop(); startedLogCacheBus = false } if (startedMetricsBus) { code.metricsstream.MetricsEventBus.stop(); startedMetricsBus = false } + if (startedSignalBus) { code.signal.SignalEventBus.stop(); startedSignalBus = false } if (server != null) { server.shutdown() server = null diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala new file mode 100644 index 0000000000..82f0c05fc8 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala @@ -0,0 +1,157 @@ +package code.obp.grpc.signal + +import code.api.cache.RedisMessaging +import code.api.util.ErrorMessages.{InvalidJsonFormat, InvalidSignalChannelName, SignalMessageContainsDangerousCharacters, SignalMessageTooLong} +import code.api.v6_0_0.{PostSignalMessageJsonV600, SignalMessageJsonV600} +import code.obp.grpc.chat.AuthInterceptor +import code.obp.grpc.signal.api._ +import code.signal.{SignalChannels, SignalContentPolicy, SignalEventBus} +import code.util.DangerousCharacters +import code.util.Helper.MdcLoggable +import com.google.protobuf.timestamp.Timestamp +import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.model.User +import com.openbankproject.commons.util.JsonAliases +import io.grpc.{Status, StatusRuntimeException} +import io.grpc.stub.{ServerCallStreamObserver, StreamObserver} +import net.liftweb.common.Full +import org.json4s.JsonAST.JValue + +import java.time.Instant +import scala.concurrent.Future +import scala.util.Try +import scala.util.control.NonFatal + +/** + * gRPC SignalChannelsService: the same four operations as the REST signal + * endpoints (publish, fetch, list) plus a live Subscribe stream, over the + * same Redis storage and the same SignalChannels helper, so a message + * published on one transport is read on the other unchanged. + * + * Auth: the shared AuthInterceptor validates the token at call open and puts + * the User in gRPC Context. Validation failures map to INVALID_ARGUMENT with + * the REST error message as the description, so a client sees the same + * OBP-xxxxx code either way. + * + * Size cap: REST caps the raw request body. The gRPC analogue is the + * JSON-encoded payload plus message_type, checked before the payload is parsed. + */ +object SignalChannelsServiceImpl extends SignalChannelsServiceGrpc.SignalChannelsService with MdcLoggable { + + private def unauthenticated: StatusRuntimeException = + Status.UNAUTHENTICATED.withDescription("Not authenticated").asRuntimeException() + + private def invalid(message: String): StatusRuntimeException = + Status.INVALID_ARGUMENT.withDescription(message).asRuntimeException() + + private def withUser[T](body: User => T): Future[T] = { + val user = AuthInterceptor.USER_CONTEXT_KEY.get() + if (user == null) Future.failed(unauthenticated) + else Future(body(user)).recoverWith { + case e: StatusRuntimeException => Future.failed(e) + case NonFatal(e) => + logger.error(s"SignalChannelsServiceImpl says: ${e.getMessage}", e) + Future.failed(Status.INTERNAL.withDescription(e.getMessage).asRuntimeException()) + } + } + + private def toTimestamp(iso: String): Option[Timestamp] = + Try(Instant.parse(iso)).toOption.map(i => Timestamp(seconds = i.getEpochSecond, nanos = i.getNano)) + + private def toProto(msg: SignalMessageJsonV600): SignalMessage = + SignalMessage( + messageId = msg.message_id, + channelName = msg.channel_name, + senderConsumerId = msg.sender_consumer_id, + senderUserId = msg.sender_user_id, + toUserId = msg.to_user_id.getOrElse(""), + timestamp = toTimestamp(msg.timestamp), + messageType = msg.message_type, + payloadJson = JsonAliases.compactRender(msg.payload), + sequence = msg.sequence) + + override def publish(request: PublishRequest): Future[PublishResponse] = withUser { user => + val callContext = Option(AuthInterceptor.CALL_CONTEXT_KEY.get()) + // Same order as the REST handler: size cap before parsing, then JSON, name, characters. + if (request.payloadJson.length + request.messageType.length > SignalContentPolicy.maxPayloadLength) + throw invalid(s"$SignalMessageTooLong Maximum: ${SignalContentPolicy.maxPayloadLength} characters.") + val payload: JValue = + try JsonAliases.parse(request.payloadJson) + catch { case NonFatal(_) => throw invalid(s"$InvalidJsonFormat payload_json must be a JSON document.") } + if (!RedisMessaging.validateChannelName(request.channelName)) throw invalid(InvalidSignalChannelName) + if (SignalContentPolicy.containsDangerousCharacters(payload) || DangerousCharacters.containsAny(request.messageType)) + throw invalid(SignalMessageContainsDangerousCharacters) + + val post = PostSignalMessageJsonV600( + payload = payload, + message_type = Option(request.messageType).filter(_.nonEmpty), + to_user_id = Option(request.toUserId).filter(_.nonEmpty)) + val consumerId = callContext.flatMap(_.consumer match { case Full(c) => Some(c.consumerId.get); case _ => None }).getOrElse("") + val published = SignalChannels.publish(request.channelName, user.userId, consumerId, post) + PublishResponse( + messageId = published.message_id, + channelName = published.channel_name, + timestamp = toTimestamp(published.timestamp), + channelMessageCount = published.channel_message_count, + sequence = published.sequence) + } + + override def fetch(request: FetchRequest): Future[FetchResponse] = withUser { user => + if (!RedisMessaging.validateChannelName(request.channelName)) throw invalid(InvalidSignalChannelName) + val offset = math.max(0, request.offset) + val limit = if (request.limit <= 0) 50 else request.limit + // proto3 cannot tell "unset" from 0, so 0 means offset mode; use offset 0 for a first read + // and continue with next_after_sequence. + val afterSequence = if (request.afterSequence > 0L) Some(request.afterSequence) else None + val page = SignalChannels.fetch(request.channelName, offset, limit, afterSequence, user.userId) + FetchResponse( + channelName = page.channel_name, + messages = page.messages.map(toProto), + totalCount = page.total_count, + hasMore = page.has_more, + latestSequence = page.latest_sequence, + nextAfterSequence = page.next_after_sequence) + } + + override def listChannels(request: ListChannelsRequest): Future[ListChannelsResponse] = withUser { _ => + ListChannelsResponse(SignalChannels.listBroadcastChannels().map(c => + SignalChannelInfo(channelName = c.channel_name, messageCount = c.message_count, ttlSeconds = c.ttl_seconds))) + } + + override def subscribe(request: SubscribeRequest, responseObserver: StreamObserver[SignalMessage]): Unit = { + val user = AuthInterceptor.USER_CONTEXT_KEY.get() + if (user == null) { + responseObserver.onError(unauthenticated) + return + } + val channelName = request.channelName + if (!RedisMessaging.validateChannelName(channelName)) { + responseObserver.onError(invalid(InvalidSignalChannelName)) + return + } + val userId = user.userId + logger.info(s"SignalChannelsServiceImpl says: User $userId subscribed to signal channel $channelName") + + val bridge = new StreamObserver[String] { + override def onNext(envelopeJson: String): Unit = + SignalChannels.parseMessage(envelopeJson) match { + case Some(msg) if SignalChannels.isVisibleTo(msg, userId) => responseObserver.onNext(toProto(msg)) + case Some(_) => // private message for someone else + case None => logger.warn(s"SignalChannelsServiceImpl says: Dropped unparseable envelope on $channelName") + } + override def onError(t: Throwable): Unit = responseObserver.onError(t) + override def onCompleted(): Unit = responseObserver.onCompleted() + } + + SignalEventBus.subscribe(channelName, bridge) + + responseObserver match { + case ssco: ServerCallStreamObserver[_] => + ssco.setOnCancelHandler(() => { + SignalEventBus.unsubscribe(channelName, bridge) + logger.info(s"SignalChannelsServiceImpl says: User $userId unsubscribed from signal channel $channelName") + }) + case _ => + } + } +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchRequest.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchRequest.scala new file mode 100644 index 0000000000..7f91d7f1cc --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchRequest.scala @@ -0,0 +1,146 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class FetchRequest( + channelName: _root_.scala.Predef.String = "", + offset: _root_.scala.Int = 0, + limit: _root_.scala.Int = 0, + afterSequence: _root_.scala.Long = 0L + ) extends scalapb.GeneratedMessage with scalapb.Message[FetchRequest] with scalapb.lenses.Updatable[FetchRequest] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, channelName) } + if (offset != 0) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt32Size(2, offset) } + if (limit != 0) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt32Size(3, limit) } + if (afterSequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(4, afterSequence) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = channelName; if (__v != "") _output__.writeString(1, __v) }; + { val __v = offset; if (__v != 0) _output__.writeInt32(2, __v) }; + { val __v = limit; if (__v != 0) _output__.writeInt32(3, __v) }; + { val __v = afterSequence; if (__v != 0L) _output__.writeInt64(4, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.FetchRequest = { + var __channelName = this.channelName + var __offset = this.offset + var __limit = this.limit + var __afterSequence = this.afterSequence + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channelName = _input__.readString() + case 16 => + __offset = _input__.readInt32() + case 24 => + __limit = _input__.readInt32() + case 32 => + __afterSequence = _input__.readInt64() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.FetchRequest( + channelName = __channelName, + offset = __offset, + limit = __limit, + afterSequence = __afterSequence + ) + } + def withChannelName(__v: _root_.scala.Predef.String): FetchRequest = copy(channelName = __v) + def withOffset(__v: _root_.scala.Int): FetchRequest = copy(offset = __v) + def withLimit(__v: _root_.scala.Int): FetchRequest = copy(limit = __v) + def withAfterSequence(__v: _root_.scala.Long): FetchRequest = copy(afterSequence = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = channelName + if (__t != "") __t else null + } + case 2 => { + val __t = offset + if (__t != 0) __t else null + } + case 3 => { + val __t = limit + if (__t != 0) __t else null + } + case 4 => { + val __t = afterSequence + if (__t != 0L) __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(channelName) + case 2 => _root_.scalapb.descriptors.PInt(offset) + case 3 => _root_.scalapb.descriptors.PInt(limit) + case 4 => _root_.scalapb.descriptors.PLong(afterSequence) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.FetchRequest +} + +object FetchRequest extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.FetchRequest] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.FetchRequest] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.FetchRequest = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.FetchRequest( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), 0).asInstanceOf[_root_.scala.Int], + __fieldsMap.getOrElse(__fields.get(2), 0).asInstanceOf[_root_.scala.Int], + __fieldsMap.getOrElse(__fields.get(3), 0L).asInstanceOf[_root_.scala.Long] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.FetchRequest] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.FetchRequest( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.Int]).getOrElse(0), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Int]).getOrElse(0), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(4) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.FetchRequest( + ) + implicit class FetchRequestLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.FetchRequest]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.FetchRequest](_l) { + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def offset: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.offset)((c_, f_) => c_.copy(offset = f_)) + def limit: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.limit)((c_, f_) => c_.copy(limit = f_)) + def afterSequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.afterSequence)((c_, f_) => c_.copy(afterSequence = f_)) + } + final val CHANNEL_NAME_FIELD_NUMBER = 1 + final val OFFSET_FIELD_NUMBER = 2 + final val LIMIT_FIELD_NUMBER = 3 + final val AFTER_SEQUENCE_FIELD_NUMBER = 4 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala new file mode 100644 index 0000000000..ef69ee7980 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala @@ -0,0 +1,190 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class FetchResponse( + channelName: _root_.scala.Predef.String = "", + messages: _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalMessage] = _root_.scala.collection.Seq.empty, + totalCount: _root_.scala.Long = 0L, + hasMore: _root_.scala.Boolean = false, + latestSequence: _root_.scala.Long = 0L, + nextAfterSequence: _root_.scala.Long = 0L + ) extends scalapb.GeneratedMessage with scalapb.Message[FetchResponse] with scalapb.lenses.Updatable[FetchResponse] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, channelName) } + messages.foreach(messages => __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(messages.serializedSize) + messages.serializedSize) + if (totalCount != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(3, totalCount) } + if (hasMore != false) { __size += _root_.com.google.protobuf.CodedOutputStream.computeBoolSize(4, hasMore) } + if (latestSequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(5, latestSequence) } + if (nextAfterSequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(6, nextAfterSequence) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = channelName; if (__v != "") _output__.writeString(1, __v) }; + messages.foreach { __v => + _output__.writeTag(2, 2) + _output__.writeUInt32NoTag(__v.serializedSize) + __v.writeTo(_output__) + }; + { val __v = totalCount; if (__v != 0L) _output__.writeInt64(3, __v) }; + { val __v = hasMore; if (__v != false) _output__.writeBool(4, __v) }; + { val __v = latestSequence; if (__v != 0L) _output__.writeInt64(5, __v) }; + { val __v = nextAfterSequence; if (__v != 0L) _output__.writeInt64(6, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.FetchResponse = { + var __channelName = this.channelName + val __messages = (_root_.scala.collection.immutable.Vector.newBuilder[code.obp.grpc.signal.api.SignalMessage] ++= this.messages) + var __totalCount = this.totalCount + var __hasMore = this.hasMore + var __latestSequence = this.latestSequence + var __nextAfterSequence = this.nextAfterSequence + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channelName = _input__.readString() + case 18 => + __messages += _root_.scalapb.LiteParser.readMessage(_input__, code.obp.grpc.signal.api.SignalMessage.defaultInstance) + case 24 => + __totalCount = _input__.readInt64() + case 32 => + __hasMore = _input__.readBool() + case 40 => + __latestSequence = _input__.readInt64() + case 48 => + __nextAfterSequence = _input__.readInt64() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.FetchResponse( + channelName = __channelName, + messages = __messages.result(), + totalCount = __totalCount, + hasMore = __hasMore, + latestSequence = __latestSequence, + nextAfterSequence = __nextAfterSequence + ) + } + def withChannelName(__v: _root_.scala.Predef.String): FetchResponse = copy(channelName = __v) + def clearMessages = copy(messages = _root_.scala.collection.Seq.empty) + def addMessages(__vs: code.obp.grpc.signal.api.SignalMessage*): FetchResponse = addAllMessages(__vs) + def addAllMessages(__vs: TraversableOnce[code.obp.grpc.signal.api.SignalMessage]): FetchResponse = copy(messages = messages ++ __vs) + def withMessages(__v: _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalMessage]): FetchResponse = copy(messages = __v) + def withTotalCount(__v: _root_.scala.Long): FetchResponse = copy(totalCount = __v) + def withHasMore(__v: _root_.scala.Boolean): FetchResponse = copy(hasMore = __v) + def withLatestSequence(__v: _root_.scala.Long): FetchResponse = copy(latestSequence = __v) + def withNextAfterSequence(__v: _root_.scala.Long): FetchResponse = copy(nextAfterSequence = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = channelName + if (__t != "") __t else null + } + case 2 => messages + case 3 => { + val __t = totalCount + if (__t != 0L) __t else null + } + case 4 => { + val __t = hasMore + if (__t != false) __t else null + } + case 5 => { + val __t = latestSequence + if (__t != 0L) __t else null + } + case 6 => { + val __t = nextAfterSequence + if (__t != 0L) __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(channelName) + case 2 => _root_.scalapb.descriptors.PRepeated(messages.iterator.map(_.toPMessage).toVector) + case 3 => _root_.scalapb.descriptors.PLong(totalCount) + case 4 => _root_.scalapb.descriptors.PBoolean(hasMore) + case 5 => _root_.scalapb.descriptors.PLong(latestSequence) + case 6 => _root_.scalapb.descriptors.PLong(nextAfterSequence) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.FetchResponse +} + +object FetchResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.FetchResponse] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.FetchResponse] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.FetchResponse = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.FetchResponse( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), Nil).asInstanceOf[_root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalMessage]], + __fieldsMap.getOrElse(__fields.get(2), 0L).asInstanceOf[_root_.scala.Long], + __fieldsMap.getOrElse(__fields.get(3), false).asInstanceOf[_root_.scala.Boolean], + __fieldsMap.getOrElse(__fields.get(4), 0L).asInstanceOf[_root_.scala.Long], + __fieldsMap.getOrElse(__fields.get(5), 0L).asInstanceOf[_root_.scala.Long] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.FetchResponse] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.FetchResponse( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalMessage]]).getOrElse(_root_.scala.collection.Seq.empty), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Long]).getOrElse(0L), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Boolean]).getOrElse(false), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Long]).getOrElse(0L), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(5) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 2 => __out = code.obp.grpc.signal.api.SignalMessage + } + __out + } + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.FetchResponse( + ) + implicit class FetchResponseLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.FetchResponse]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.FetchResponse](_l) { + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def messages: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalMessage]] = field(_.messages)((c_, f_) => c_.copy(messages = f_)) + def totalCount: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.totalCount)((c_, f_) => c_.copy(totalCount = f_)) + def hasMore: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Boolean] = field(_.hasMore)((c_, f_) => c_.copy(hasMore = f_)) + def latestSequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.latestSequence)((c_, f_) => c_.copy(latestSequence = f_)) + def nextAfterSequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.nextAfterSequence)((c_, f_) => c_.copy(nextAfterSequence = f_)) + } + final val CHANNEL_NAME_FIELD_NUMBER = 1 + final val MESSAGES_FIELD_NUMBER = 2 + final val TOTAL_COUNT_FIELD_NUMBER = 3 + final val HAS_MORE_FIELD_NUMBER = 4 + final val LATEST_SEQUENCE_FIELD_NUMBER = 5 + final val NEXT_AFTER_SEQUENCE_FIELD_NUMBER = 6 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsRequest.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsRequest.scala new file mode 100644 index 0000000000..b179608dde --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsRequest.scala @@ -0,0 +1,86 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class ListChannelsRequest( + + ) extends scalapb.GeneratedMessage with scalapb.Message[ListChannelsRequest] with scalapb.lenses.Updatable[ListChannelsRequest] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.ListChannelsRequest = { + + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.ListChannelsRequest( + + ) + } + + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + throw new MatchError(__fieldNumber) + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + throw new MatchError(__field.number) + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.ListChannelsRequest +} + +object ListChannelsRequest extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.ListChannelsRequest] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.ListChannelsRequest] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.ListChannelsRequest = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.ListChannelsRequest( + + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.ListChannelsRequest] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.ListChannelsRequest( + + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(6) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.ListChannelsRequest( + ) + implicit class ListChannelsRequestLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.ListChannelsRequest]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.ListChannelsRequest](_l) { + + } + +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsResponse.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsResponse.scala new file mode 100644 index 0000000000..e9f3e9290d --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/ListChannelsResponse.scala @@ -0,0 +1,105 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class ListChannelsResponse( + channels: _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalChannelInfo] = _root_.scala.collection.Seq.empty + ) extends scalapb.GeneratedMessage with scalapb.Message[ListChannelsResponse] with scalapb.lenses.Updatable[ListChannelsResponse] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + channels.foreach(channels => __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(channels.serializedSize) + channels.serializedSize) + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + channels.foreach { __v => + _output__.writeTag(1, 2) + _output__.writeUInt32NoTag(__v.serializedSize) + __v.writeTo(_output__) + }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.ListChannelsResponse = { + val __channels = (_root_.scala.collection.immutable.Vector.newBuilder[code.obp.grpc.signal.api.SignalChannelInfo] ++= this.channels) + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channels += _root_.scalapb.LiteParser.readMessage(_input__, code.obp.grpc.signal.api.SignalChannelInfo.defaultInstance) + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.ListChannelsResponse( + channels = __channels.result() + ) + } + def clearChannels = copy(channels = _root_.scala.collection.Seq.empty) + def addChannels(__vs: code.obp.grpc.signal.api.SignalChannelInfo*): ListChannelsResponse = addAllChannels(__vs) + def addAllChannels(__vs: TraversableOnce[code.obp.grpc.signal.api.SignalChannelInfo]): ListChannelsResponse = copy(channels = channels ++ __vs) + def withChannels(__v: _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalChannelInfo]): ListChannelsResponse = copy(channels = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => channels + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PRepeated(channels.iterator.map(_.toPMessage).toVector) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.ListChannelsResponse +} + +object ListChannelsResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.ListChannelsResponse] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.ListChannelsResponse] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.ListChannelsResponse = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.ListChannelsResponse( + __fieldsMap.getOrElse(__fields.get(0), Nil).asInstanceOf[_root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalChannelInfo]] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.ListChannelsResponse] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.ListChannelsResponse( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalChannelInfo]]).getOrElse(_root_.scala.collection.Seq.empty) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(7) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 1 => __out = code.obp.grpc.signal.api.SignalChannelInfo + } + __out + } + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.ListChannelsResponse( + ) + implicit class ListChannelsResponseLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.ListChannelsResponse]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.ListChannelsResponse](_l) { + def channels: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.collection.Seq[code.obp.grpc.signal.api.SignalChannelInfo]] = field(_.channels)((c_, f_) => c_.copy(channels = f_)) + } + final val CHANNELS_FIELD_NUMBER = 1 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishRequest.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishRequest.scala new file mode 100644 index 0000000000..f2641c3ad6 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishRequest.scala @@ -0,0 +1,146 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class PublishRequest( + channelName: _root_.scala.Predef.String = "", + toUserId: _root_.scala.Predef.String = "", + messageType: _root_.scala.Predef.String = "", + payloadJson: _root_.scala.Predef.String = "" + ) extends scalapb.GeneratedMessage with scalapb.Message[PublishRequest] with scalapb.lenses.Updatable[PublishRequest] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, channelName) } + if (toUserId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(2, toUserId) } + if (messageType != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(3, messageType) } + if (payloadJson != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(4, payloadJson) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = channelName; if (__v != "") _output__.writeString(1, __v) }; + { val __v = toUserId; if (__v != "") _output__.writeString(2, __v) }; + { val __v = messageType; if (__v != "") _output__.writeString(3, __v) }; + { val __v = payloadJson; if (__v != "") _output__.writeString(4, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.PublishRequest = { + var __channelName = this.channelName + var __toUserId = this.toUserId + var __messageType = this.messageType + var __payloadJson = this.payloadJson + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channelName = _input__.readString() + case 18 => + __toUserId = _input__.readString() + case 26 => + __messageType = _input__.readString() + case 34 => + __payloadJson = _input__.readString() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.PublishRequest( + channelName = __channelName, + toUserId = __toUserId, + messageType = __messageType, + payloadJson = __payloadJson + ) + } + def withChannelName(__v: _root_.scala.Predef.String): PublishRequest = copy(channelName = __v) + def withToUserId(__v: _root_.scala.Predef.String): PublishRequest = copy(toUserId = __v) + def withMessageType(__v: _root_.scala.Predef.String): PublishRequest = copy(messageType = __v) + def withPayloadJson(__v: _root_.scala.Predef.String): PublishRequest = copy(payloadJson = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = channelName + if (__t != "") __t else null + } + case 2 => { + val __t = toUserId + if (__t != "") __t else null + } + case 3 => { + val __t = messageType + if (__t != "") __t else null + } + case 4 => { + val __t = payloadJson + if (__t != "") __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(channelName) + case 2 => _root_.scalapb.descriptors.PString(toUserId) + case 3 => _root_.scalapb.descriptors.PString(messageType) + case 4 => _root_.scalapb.descriptors.PString(payloadJson) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.PublishRequest +} + +object PublishRequest extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.PublishRequest] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.PublishRequest] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.PublishRequest = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.PublishRequest( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(2), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(3), "").asInstanceOf[_root_.scala.Predef.String] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.PublishRequest] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.PublishRequest( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(2) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.PublishRequest( + ) + implicit class PublishRequestLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.PublishRequest]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.PublishRequest](_l) { + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def toUserId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.toUserId)((c_, f_) => c_.copy(toUserId = f_)) + def messageType: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.messageType)((c_, f_) => c_.copy(messageType = f_)) + def payloadJson: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.payloadJson)((c_, f_) => c_.copy(payloadJson = f_)) + } + final val CHANNEL_NAME_FIELD_NUMBER = 1 + final val TO_USER_ID_FIELD_NUMBER = 2 + final val MESSAGE_TYPE_FIELD_NUMBER = 3 + final val PAYLOAD_JSON_FIELD_NUMBER = 4 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishResponse.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishResponse.scala new file mode 100644 index 0000000000..339aa94bd2 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/PublishResponse.scala @@ -0,0 +1,177 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class PublishResponse( + messageId: _root_.scala.Predef.String = "", + channelName: _root_.scala.Predef.String = "", + timestamp: _root_.scala.Option[com.google.protobuf.timestamp.Timestamp] = _root_.scala.None, + channelMessageCount: _root_.scala.Long = 0L, + sequence: _root_.scala.Long = 0L + ) extends scalapb.GeneratedMessage with scalapb.Message[PublishResponse] with scalapb.lenses.Updatable[PublishResponse] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (messageId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, messageId) } + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(2, channelName) } + if (timestamp.isDefined) { + val __v = timestamp.get + val __s = __v.serializedSize + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__s) + __s + } + if (channelMessageCount != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(4, channelMessageCount) } + if (sequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(5, sequence) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = messageId; if (__v != "") _output__.writeString(1, __v) }; + { val __v = channelName; if (__v != "") _output__.writeString(2, __v) }; + timestamp.foreach { __v => + _output__.writeTag(3, 2) + _output__.writeUInt32NoTag(__v.serializedSize) + __v.writeTo(_output__) + }; + { val __v = channelMessageCount; if (__v != 0L) _output__.writeInt64(4, __v) }; + { val __v = sequence; if (__v != 0L) _output__.writeInt64(5, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.PublishResponse = { + var __messageId = this.messageId + var __channelName = this.channelName + var __timestamp = this.timestamp + var __channelMessageCount = this.channelMessageCount + var __sequence = this.sequence + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __messageId = _input__.readString() + case 18 => + __channelName = _input__.readString() + case 26 => + __timestamp = Some(_root_.scalapb.LiteParser.readMessage(_input__, __timestamp.getOrElse(com.google.protobuf.timestamp.Timestamp.defaultInstance))) + case 32 => + __channelMessageCount = _input__.readInt64() + case 40 => + __sequence = _input__.readInt64() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.PublishResponse( + messageId = __messageId, + channelName = __channelName, + timestamp = __timestamp, + channelMessageCount = __channelMessageCount, + sequence = __sequence + ) + } + def withMessageId(__v: _root_.scala.Predef.String): PublishResponse = copy(messageId = __v) + def withChannelName(__v: _root_.scala.Predef.String): PublishResponse = copy(channelName = __v) + def getTimestamp: com.google.protobuf.timestamp.Timestamp = timestamp.getOrElse(com.google.protobuf.timestamp.Timestamp.defaultInstance) + def clearTimestamp: PublishResponse = copy(timestamp = _root_.scala.None) + def withTimestamp(__v: com.google.protobuf.timestamp.Timestamp): PublishResponse = copy(timestamp = Some(__v)) + def withChannelMessageCount(__v: _root_.scala.Long): PublishResponse = copy(channelMessageCount = __v) + def withSequence(__v: _root_.scala.Long): PublishResponse = copy(sequence = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = messageId + if (__t != "") __t else null + } + case 2 => { + val __t = channelName + if (__t != "") __t else null + } + case 3 => timestamp.orNull + case 4 => { + val __t = channelMessageCount + if (__t != 0L) __t else null + } + case 5 => { + val __t = sequence + if (__t != 0L) __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(messageId) + case 2 => _root_.scalapb.descriptors.PString(channelName) + case 3 => timestamp.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) + case 4 => _root_.scalapb.descriptors.PLong(channelMessageCount) + case 5 => _root_.scalapb.descriptors.PLong(sequence) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.PublishResponse +} + +object PublishResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.PublishResponse] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.PublishResponse] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.PublishResponse = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.PublishResponse( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.get(__fields.get(2)).asInstanceOf[_root_.scala.Option[com.google.protobuf.timestamp.Timestamp]], + __fieldsMap.getOrElse(__fields.get(3), 0L).asInstanceOf[_root_.scala.Long], + __fieldsMap.getOrElse(__fields.get(4), 0L).asInstanceOf[_root_.scala.Long] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.PublishResponse] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.PublishResponse( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).flatMap(_.as[_root_.scala.Option[com.google.protobuf.timestamp.Timestamp]]), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Long]).getOrElse(0L), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(3) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 3 => __out = com.google.protobuf.timestamp.Timestamp + } + __out + } + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.PublishResponse( + ) + implicit class PublishResponseLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.PublishResponse]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.PublishResponse](_l) { + def messageId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.messageId)((c_, f_) => c_.copy(messageId = f_)) + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def timestamp: _root_.scalapb.lenses.Lens[UpperPB, com.google.protobuf.timestamp.Timestamp] = field(_.getTimestamp)((c_, f_) => c_.copy(timestamp = Some(f_))) + def optionalTimestamp: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[com.google.protobuf.timestamp.Timestamp]] = field(_.timestamp)((c_, f_) => c_.copy(timestamp = f_)) + def channelMessageCount: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.channelMessageCount)((c_, f_) => c_.copy(channelMessageCount = f_)) + def sequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.sequence)((c_, f_) => c_.copy(sequence = f_)) + } + final val MESSAGE_ID_FIELD_NUMBER = 1 + final val CHANNEL_NAME_FIELD_NUMBER = 2 + final val TIMESTAMP_FIELD_NUMBER = 3 + final val CHANNEL_MESSAGE_COUNT_FIELD_NUMBER = 4 + final val SEQUENCE_FIELD_NUMBER = 5 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelInfo.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelInfo.scala new file mode 100644 index 0000000000..9fcebcc242 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelInfo.scala @@ -0,0 +1,129 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class SignalChannelInfo( + channelName: _root_.scala.Predef.String = "", + messageCount: _root_.scala.Long = 0L, + ttlSeconds: _root_.scala.Long = 0L + ) extends scalapb.GeneratedMessage with scalapb.Message[SignalChannelInfo] with scalapb.lenses.Updatable[SignalChannelInfo] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, channelName) } + if (messageCount != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(2, messageCount) } + if (ttlSeconds != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(3, ttlSeconds) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = channelName; if (__v != "") _output__.writeString(1, __v) }; + { val __v = messageCount; if (__v != 0L) _output__.writeInt64(2, __v) }; + { val __v = ttlSeconds; if (__v != 0L) _output__.writeInt64(3, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.SignalChannelInfo = { + var __channelName = this.channelName + var __messageCount = this.messageCount + var __ttlSeconds = this.ttlSeconds + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channelName = _input__.readString() + case 16 => + __messageCount = _input__.readInt64() + case 24 => + __ttlSeconds = _input__.readInt64() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.SignalChannelInfo( + channelName = __channelName, + messageCount = __messageCount, + ttlSeconds = __ttlSeconds + ) + } + def withChannelName(__v: _root_.scala.Predef.String): SignalChannelInfo = copy(channelName = __v) + def withMessageCount(__v: _root_.scala.Long): SignalChannelInfo = copy(messageCount = __v) + def withTtlSeconds(__v: _root_.scala.Long): SignalChannelInfo = copy(ttlSeconds = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = channelName + if (__t != "") __t else null + } + case 2 => { + val __t = messageCount + if (__t != 0L) __t else null + } + case 3 => { + val __t = ttlSeconds + if (__t != 0L) __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(channelName) + case 2 => _root_.scalapb.descriptors.PLong(messageCount) + case 3 => _root_.scalapb.descriptors.PLong(ttlSeconds) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.SignalChannelInfo +} + +object SignalChannelInfo extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SignalChannelInfo] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SignalChannelInfo] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.SignalChannelInfo = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.SignalChannelInfo( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), 0L).asInstanceOf[_root_.scala.Long], + __fieldsMap.getOrElse(__fields.get(2), 0L).asInstanceOf[_root_.scala.Long] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.SignalChannelInfo] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.SignalChannelInfo( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.Long]).getOrElse(0L), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(1) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.SignalChannelInfo( + ) + implicit class SignalChannelInfoLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.SignalChannelInfo]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.SignalChannelInfo](_l) { + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def messageCount: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.messageCount)((c_, f_) => c_.copy(messageCount = f_)) + def ttlSeconds: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.ttlSeconds)((c_, f_) => c_.copy(ttlSeconds = f_)) + } + final val CHANNEL_NAME_FIELD_NUMBER = 1 + final val MESSAGE_COUNT_FIELD_NUMBER = 2 + final val TTL_SECONDS_FIELD_NUMBER = 3 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala new file mode 100644 index 0000000000..f7307c3660 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala @@ -0,0 +1,163 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api/ChatStreamServiceGrpc.scala and +// api/ObpServiceGrpc.scala). No protoc plugin is wired into the Maven build. +// Source of truth: obp-api/src/main/protobuf/signal.proto. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +object SignalChannelsServiceGrpc { + + private val SERVICE_NAME = "code.obp.grpc.signal.g1.SignalChannelsService" + + val METHOD_PUBLISH: _root_.io.grpc.MethodDescriptor[PublishRequest, PublishResponse] = + _root_.io.grpc.MethodDescriptor.newBuilder() + .setType(_root_.io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(_root_.io.grpc.MethodDescriptor.generateFullMethodName(SERVICE_NAME, "Publish")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(new scalapb.grpc.Marshaller(PublishRequest)) + .setResponseMarshaller(new scalapb.grpc.Marshaller(PublishResponse)) + .build() + + val METHOD_FETCH: _root_.io.grpc.MethodDescriptor[FetchRequest, FetchResponse] = + _root_.io.grpc.MethodDescriptor.newBuilder() + .setType(_root_.io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(_root_.io.grpc.MethodDescriptor.generateFullMethodName(SERVICE_NAME, "Fetch")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(new scalapb.grpc.Marshaller(FetchRequest)) + .setResponseMarshaller(new scalapb.grpc.Marshaller(FetchResponse)) + .build() + + val METHOD_LIST_CHANNELS: _root_.io.grpc.MethodDescriptor[ListChannelsRequest, ListChannelsResponse] = + _root_.io.grpc.MethodDescriptor.newBuilder() + .setType(_root_.io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(_root_.io.grpc.MethodDescriptor.generateFullMethodName(SERVICE_NAME, "ListChannels")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(new scalapb.grpc.Marshaller(ListChannelsRequest)) + .setResponseMarshaller(new scalapb.grpc.Marshaller(ListChannelsResponse)) + .build() + + val METHOD_SUBSCRIBE: _root_.io.grpc.MethodDescriptor[SubscribeRequest, SignalMessage] = + _root_.io.grpc.MethodDescriptor.newBuilder() + .setType(_root_.io.grpc.MethodDescriptor.MethodType.SERVER_STREAMING) + .setFullMethodName(_root_.io.grpc.MethodDescriptor.generateFullMethodName(SERVICE_NAME, "Subscribe")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(new scalapb.grpc.Marshaller(SubscribeRequest)) + .setResponseMarshaller(new scalapb.grpc.Marshaller(SignalMessage)) + .build() + + val SERVICE: _root_.io.grpc.ServiceDescriptor = + _root_.io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) + .setSchemaDescriptor(new _root_.scalapb.grpc.ConcreteProtoFileDescriptorSupplier(SignalProto.javaDescriptor)) + .addMethod(METHOD_PUBLISH) + .addMethod(METHOD_FETCH) + .addMethod(METHOD_LIST_CHANNELS) + .addMethod(METHOD_SUBSCRIBE) + .build() + + trait SignalChannelsService extends _root_.scalapb.grpc.AbstractService { + override def serviceCompanion = SignalChannelsService + + /** 1:1 with POST /signal/channels/CHANNEL_NAME/messages */ + def publish(request: PublishRequest): scala.concurrent.Future[PublishResponse] + + /** 1:1 with GET /signal/channels/CHANNEL_NAME/messages (offset/limit, privacy-filtered) */ + def fetch(request: FetchRequest): scala.concurrent.Future[FetchResponse] + + /** 1:1 with GET /signal/channels (broadcast-visible channels only) */ + def listChannels(request: ListChannelsRequest): scala.concurrent.Future[ListChannelsResponse] + + /** Server-side stream of new messages on one channel. Live only: no catch-up, no replay. */ + def subscribe(request: SubscribeRequest, + responseObserver: _root_.io.grpc.stub.StreamObserver[SignalMessage]): Unit + } + + object SignalChannelsService extends _root_.scalapb.grpc.ServiceCompanion[SignalChannelsService] { + implicit def serviceCompanion: _root_.scalapb.grpc.ServiceCompanion[SignalChannelsService] = this + def javaDescriptor: _root_.com.google.protobuf.Descriptors.ServiceDescriptor = + SignalProto.javaDescriptor.getServices().get(0) + } + + trait SignalChannelsServiceBlockingClient { + def serviceCompanion = SignalChannelsService + def publish(request: PublishRequest): PublishResponse + def fetch(request: FetchRequest): FetchResponse + def listChannels(request: ListChannelsRequest): ListChannelsResponse + def subscribe(request: SubscribeRequest): scala.collection.Iterator[SignalMessage] + } + + class SignalChannelsServiceBlockingStub(channel: _root_.io.grpc.Channel, options: _root_.io.grpc.CallOptions = _root_.io.grpc.CallOptions.DEFAULT) + extends _root_.io.grpc.stub.AbstractStub[SignalChannelsServiceBlockingStub](channel, options) with SignalChannelsServiceBlockingClient { + + override def publish(request: PublishRequest): PublishResponse = + _root_.io.grpc.stub.ClientCalls.blockingUnaryCall(channel.newCall(METHOD_PUBLISH, options), request) + + override def fetch(request: FetchRequest): FetchResponse = + _root_.io.grpc.stub.ClientCalls.blockingUnaryCall(channel.newCall(METHOD_FETCH, options), request) + + override def listChannels(request: ListChannelsRequest): ListChannelsResponse = + _root_.io.grpc.stub.ClientCalls.blockingUnaryCall(channel.newCall(METHOD_LIST_CHANNELS, options), request) + + override def subscribe(request: SubscribeRequest): scala.collection.Iterator[SignalMessage] = + scala.jdk.CollectionConverters.IteratorHasAsScala( + _root_.io.grpc.stub.ClientCalls.blockingServerStreamingCall(channel.newCall(METHOD_SUBSCRIBE, options), request)).asScala + + override def build(channel: _root_.io.grpc.Channel, options: _root_.io.grpc.CallOptions): SignalChannelsServiceBlockingStub = + new SignalChannelsServiceBlockingStub(channel, options) + } + + class SignalChannelsServiceStub(channel: _root_.io.grpc.Channel, options: _root_.io.grpc.CallOptions = _root_.io.grpc.CallOptions.DEFAULT) + extends _root_.io.grpc.stub.AbstractStub[SignalChannelsServiceStub](channel, options) with SignalChannelsService { + + override def publish(request: PublishRequest): scala.concurrent.Future[PublishResponse] = + scalapb.grpc.Grpc.guavaFuture2ScalaFuture(_root_.io.grpc.stub.ClientCalls.futureUnaryCall(channel.newCall(METHOD_PUBLISH, options), request)) + + override def fetch(request: FetchRequest): scala.concurrent.Future[FetchResponse] = + scalapb.grpc.Grpc.guavaFuture2ScalaFuture(_root_.io.grpc.stub.ClientCalls.futureUnaryCall(channel.newCall(METHOD_FETCH, options), request)) + + override def listChannels(request: ListChannelsRequest): scala.concurrent.Future[ListChannelsResponse] = + scalapb.grpc.Grpc.guavaFuture2ScalaFuture(_root_.io.grpc.stub.ClientCalls.futureUnaryCall(channel.newCall(METHOD_LIST_CHANNELS, options), request)) + + override def subscribe(request: SubscribeRequest, responseObserver: _root_.io.grpc.stub.StreamObserver[SignalMessage]): Unit = + _root_.io.grpc.stub.ClientCalls.asyncServerStreamingCall(channel.newCall(METHOD_SUBSCRIBE, options), request, responseObserver) + + override def build(channel: _root_.io.grpc.Channel, options: _root_.io.grpc.CallOptions): SignalChannelsServiceStub = + new SignalChannelsServiceStub(channel, options) + } + + def bindService(serviceImpl: SignalChannelsService, executionContext: scala.concurrent.ExecutionContext): _root_.io.grpc.ServerServiceDefinition = + _root_.io.grpc.ServerServiceDefinition.builder(SERVICE) + .addMethod( + METHOD_PUBLISH, + _root_.io.grpc.stub.ServerCalls.asyncUnaryCall(new _root_.io.grpc.stub.ServerCalls.UnaryMethod[PublishRequest, PublishResponse] { + override def invoke(request: PublishRequest, observer: _root_.io.grpc.stub.StreamObserver[PublishResponse]): Unit = + serviceImpl.publish(request).onComplete(scalapb.grpc.Grpc.completeObserver(observer))(executionContext) + })) + .addMethod( + METHOD_FETCH, + _root_.io.grpc.stub.ServerCalls.asyncUnaryCall(new _root_.io.grpc.stub.ServerCalls.UnaryMethod[FetchRequest, FetchResponse] { + override def invoke(request: FetchRequest, observer: _root_.io.grpc.stub.StreamObserver[FetchResponse]): Unit = + serviceImpl.fetch(request).onComplete(scalapb.grpc.Grpc.completeObserver(observer))(executionContext) + })) + .addMethod( + METHOD_LIST_CHANNELS, + _root_.io.grpc.stub.ServerCalls.asyncUnaryCall(new _root_.io.grpc.stub.ServerCalls.UnaryMethod[ListChannelsRequest, ListChannelsResponse] { + override def invoke(request: ListChannelsRequest, observer: _root_.io.grpc.stub.StreamObserver[ListChannelsResponse]): Unit = + serviceImpl.listChannels(request).onComplete(scalapb.grpc.Grpc.completeObserver(observer))(executionContext) + })) + .addMethod( + METHOD_SUBSCRIBE, + _root_.io.grpc.stub.ServerCalls.asyncServerStreamingCall( + new _root_.io.grpc.stub.ServerCalls.ServerStreamingMethod[SubscribeRequest, SignalMessage] { + override def invoke(request: SubscribeRequest, responseObserver: _root_.io.grpc.stub.StreamObserver[SignalMessage]): Unit = + serviceImpl.subscribe(request, responseObserver) + })) + .build() + + def blockingStub(channel: _root_.io.grpc.Channel): SignalChannelsServiceBlockingStub = new SignalChannelsServiceBlockingStub(channel) + + def stub(channel: _root_.io.grpc.Channel): SignalChannelsServiceStub = new SignalChannelsServiceStub(channel) + + def javaDescriptor: _root_.com.google.protobuf.Descriptors.ServiceDescriptor = SignalProto.javaDescriptor.getServices().get(0) +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalMessage.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalMessage.scala new file mode 100644 index 0000000000..382e71c746 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalMessage.scala @@ -0,0 +1,245 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class SignalMessage( + messageId: _root_.scala.Predef.String = "", + channelName: _root_.scala.Predef.String = "", + senderConsumerId: _root_.scala.Predef.String = "", + senderUserId: _root_.scala.Predef.String = "", + toUserId: _root_.scala.Predef.String = "", + timestamp: _root_.scala.Option[com.google.protobuf.timestamp.Timestamp] = _root_.scala.None, + messageType: _root_.scala.Predef.String = "", + payloadJson: _root_.scala.Predef.String = "", + sequence: _root_.scala.Long = 0L + ) extends scalapb.GeneratedMessage with scalapb.Message[SignalMessage] with scalapb.lenses.Updatable[SignalMessage] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (messageId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, messageId) } + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(2, channelName) } + if (senderConsumerId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(3, senderConsumerId) } + if (senderUserId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(4, senderUserId) } + if (toUserId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(5, toUserId) } + if (timestamp.isDefined) { + val __v = timestamp.get + val __s = __v.serializedSize + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__s) + __s + } + if (messageType != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, messageType) } + if (payloadJson != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(8, payloadJson) } + if (sequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(9, sequence) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = messageId; if (__v != "") _output__.writeString(1, __v) }; + { val __v = channelName; if (__v != "") _output__.writeString(2, __v) }; + { val __v = senderConsumerId; if (__v != "") _output__.writeString(3, __v) }; + { val __v = senderUserId; if (__v != "") _output__.writeString(4, __v) }; + { val __v = toUserId; if (__v != "") _output__.writeString(5, __v) }; + timestamp.foreach { __v => + _output__.writeTag(6, 2) + _output__.writeUInt32NoTag(__v.serializedSize) + __v.writeTo(_output__) + }; + { val __v = messageType; if (__v != "") _output__.writeString(7, __v) }; + { val __v = payloadJson; if (__v != "") _output__.writeString(8, __v) }; + { val __v = sequence; if (__v != 0L) _output__.writeInt64(9, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.SignalMessage = { + var __messageId = this.messageId + var __channelName = this.channelName + var __senderConsumerId = this.senderConsumerId + var __senderUserId = this.senderUserId + var __toUserId = this.toUserId + var __timestamp = this.timestamp + var __messageType = this.messageType + var __payloadJson = this.payloadJson + var __sequence = this.sequence + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __messageId = _input__.readString() + case 18 => + __channelName = _input__.readString() + case 26 => + __senderConsumerId = _input__.readString() + case 34 => + __senderUserId = _input__.readString() + case 42 => + __toUserId = _input__.readString() + case 50 => + __timestamp = Some(_root_.scalapb.LiteParser.readMessage(_input__, __timestamp.getOrElse(com.google.protobuf.timestamp.Timestamp.defaultInstance))) + case 58 => + __messageType = _input__.readString() + case 66 => + __payloadJson = _input__.readString() + case 72 => + __sequence = _input__.readInt64() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.SignalMessage( + messageId = __messageId, + channelName = __channelName, + senderConsumerId = __senderConsumerId, + senderUserId = __senderUserId, + toUserId = __toUserId, + timestamp = __timestamp, + messageType = __messageType, + payloadJson = __payloadJson, + sequence = __sequence + ) + } + def withMessageId(__v: _root_.scala.Predef.String): SignalMessage = copy(messageId = __v) + def withChannelName(__v: _root_.scala.Predef.String): SignalMessage = copy(channelName = __v) + def withSenderConsumerId(__v: _root_.scala.Predef.String): SignalMessage = copy(senderConsumerId = __v) + def withSenderUserId(__v: _root_.scala.Predef.String): SignalMessage = copy(senderUserId = __v) + def withToUserId(__v: _root_.scala.Predef.String): SignalMessage = copy(toUserId = __v) + def getTimestamp: com.google.protobuf.timestamp.Timestamp = timestamp.getOrElse(com.google.protobuf.timestamp.Timestamp.defaultInstance) + def clearTimestamp: SignalMessage = copy(timestamp = _root_.scala.None) + def withTimestamp(__v: com.google.protobuf.timestamp.Timestamp): SignalMessage = copy(timestamp = Some(__v)) + def withMessageType(__v: _root_.scala.Predef.String): SignalMessage = copy(messageType = __v) + def withPayloadJson(__v: _root_.scala.Predef.String): SignalMessage = copy(payloadJson = __v) + def withSequence(__v: _root_.scala.Long): SignalMessage = copy(sequence = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = messageId + if (__t != "") __t else null + } + case 2 => { + val __t = channelName + if (__t != "") __t else null + } + case 3 => { + val __t = senderConsumerId + if (__t != "") __t else null + } + case 4 => { + val __t = senderUserId + if (__t != "") __t else null + } + case 5 => { + val __t = toUserId + if (__t != "") __t else null + } + case 6 => timestamp.orNull + case 7 => { + val __t = messageType + if (__t != "") __t else null + } + case 8 => { + val __t = payloadJson + if (__t != "") __t else null + } + case 9 => { + val __t = sequence + if (__t != 0L) __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(messageId) + case 2 => _root_.scalapb.descriptors.PString(channelName) + case 3 => _root_.scalapb.descriptors.PString(senderConsumerId) + case 4 => _root_.scalapb.descriptors.PString(senderUserId) + case 5 => _root_.scalapb.descriptors.PString(toUserId) + case 6 => timestamp.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) + case 7 => _root_.scalapb.descriptors.PString(messageType) + case 8 => _root_.scalapb.descriptors.PString(payloadJson) + case 9 => _root_.scalapb.descriptors.PLong(sequence) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.SignalMessage +} + +object SignalMessage extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SignalMessage] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SignalMessage] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.SignalMessage = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.SignalMessage( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(1), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(2), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(3), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(4), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.get(__fields.get(5)).asInstanceOf[_root_.scala.Option[com.google.protobuf.timestamp.Timestamp]], + __fieldsMap.getOrElse(__fields.get(6), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(7), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(8), 0L).asInstanceOf[_root_.scala.Long] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.SignalMessage] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.SignalMessage( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).flatMap(_.as[_root_.scala.Option[com.google.protobuf.timestamp.Timestamp]]), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(8).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(9).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(0) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 6 => __out = com.google.protobuf.timestamp.Timestamp + } + __out + } + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.SignalMessage( + ) + implicit class SignalMessageLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.SignalMessage]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.SignalMessage](_l) { + def messageId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.messageId)((c_, f_) => c_.copy(messageId = f_)) + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + def senderConsumerId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.senderConsumerId)((c_, f_) => c_.copy(senderConsumerId = f_)) + def senderUserId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.senderUserId)((c_, f_) => c_.copy(senderUserId = f_)) + def toUserId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.toUserId)((c_, f_) => c_.copy(toUserId = f_)) + def timestamp: _root_.scalapb.lenses.Lens[UpperPB, com.google.protobuf.timestamp.Timestamp] = field(_.getTimestamp)((c_, f_) => c_.copy(timestamp = Some(f_))) + def optionalTimestamp: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[com.google.protobuf.timestamp.Timestamp]] = field(_.timestamp)((c_, f_) => c_.copy(timestamp = f_)) + def messageType: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.messageType)((c_, f_) => c_.copy(messageType = f_)) + def payloadJson: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.payloadJson)((c_, f_) => c_.copy(payloadJson = f_)) + def sequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.sequence)((c_, f_) => c_.copy(sequence = f_)) + } + final val MESSAGE_ID_FIELD_NUMBER = 1 + final val CHANNEL_NAME_FIELD_NUMBER = 2 + final val SENDER_CONSUMER_ID_FIELD_NUMBER = 3 + final val SENDER_USER_ID_FIELD_NUMBER = 4 + final val TO_USER_ID_FIELD_NUMBER = 5 + final val TIMESTAMP_FIELD_NUMBER = 6 + final val MESSAGE_TYPE_FIELD_NUMBER = 7 + final val PAYLOAD_JSON_FIELD_NUMBER = 8 + final val SEQUENCE_FIELD_NUMBER = 9 +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala new file mode 100644 index 0000000000..cb81ff5b85 --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala @@ -0,0 +1,159 @@ +package code.obp.grpc.signal.api + +import com.google.protobuf.DescriptorProtos._ +import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.{Label, Type} + +/** + * Proto file descriptor for the signal channels service, built programmatically + * so gRPC reflection (service discovery) works without a protoc plugin in the + * Maven build. Must stay in step with obp-api/src/main/protobuf/signal.proto: + * message order here is the index the companions use in `javaDescriptor`. + */ +object SignalProto { + + lazy val javaDescriptor: com.google.protobuf.Descriptors.FileDescriptor = { + val fileProto = FileDescriptorProto.newBuilder() + .setName("signal.proto") + .setPackage("code.obp.grpc.signal.g1") + .setSyntax("proto3") + .addDependency("google/protobuf/timestamp.proto") + // 0: SignalMessage + .addMessageType(DescriptorProto.newBuilder() + .setName("SignalMessage") + .addField(stringField("message_id", 1)) + .addField(stringField("channel_name", 2)) + .addField(stringField("sender_consumer_id", 3)) + .addField(stringField("sender_user_id", 4)) + .addField(stringField("to_user_id", 5)) + .addField(messageField("timestamp", 6, ".google.protobuf.Timestamp")) + .addField(stringField("message_type", 7)) + .addField(stringField("payload_json", 8)) + .addField(int64Field("sequence", 9)) + ) + // 1: SignalChannelInfo + .addMessageType(DescriptorProto.newBuilder() + .setName("SignalChannelInfo") + .addField(stringField("channel_name", 1)) + .addField(int64Field("message_count", 2)) + .addField(int64Field("ttl_seconds", 3)) + ) + // 2: PublishRequest + .addMessageType(DescriptorProto.newBuilder() + .setName("PublishRequest") + .addField(stringField("channel_name", 1)) + .addField(stringField("to_user_id", 2)) + .addField(stringField("message_type", 3)) + .addField(stringField("payload_json", 4)) + ) + // 3: PublishResponse + .addMessageType(DescriptorProto.newBuilder() + .setName("PublishResponse") + .addField(stringField("message_id", 1)) + .addField(stringField("channel_name", 2)) + .addField(messageField("timestamp", 3, ".google.protobuf.Timestamp")) + .addField(int64Field("channel_message_count", 4)) + .addField(int64Field("sequence", 5)) + ) + // 4: FetchRequest + .addMessageType(DescriptorProto.newBuilder() + .setName("FetchRequest") + .addField(stringField("channel_name", 1)) + .addField(int32Field("offset", 2)) + .addField(int32Field("limit", 3)) + .addField(int64Field("after_sequence", 4)) + ) + // 5: FetchResponse + .addMessageType(DescriptorProto.newBuilder() + .setName("FetchResponse") + .addField(stringField("channel_name", 1)) + .addField(repeatedMessageField("messages", 2, ".code.obp.grpc.signal.g1.SignalMessage")) + .addField(int64Field("total_count", 3)) + .addField(boolField("has_more", 4)) + .addField(int64Field("latest_sequence", 5)) + .addField(int64Field("next_after_sequence", 6)) + ) + // 6: ListChannelsRequest + .addMessageType(DescriptorProto.newBuilder() + .setName("ListChannelsRequest") + ) + // 7: ListChannelsResponse + .addMessageType(DescriptorProto.newBuilder() + .setName("ListChannelsResponse") + .addField(repeatedMessageField("channels", 1, ".code.obp.grpc.signal.g1.SignalChannelInfo")) + ) + // 8: SubscribeRequest + .addMessageType(DescriptorProto.newBuilder() + .setName("SubscribeRequest") + .addField(stringField("channel_name", 1)) + ) + // SignalChannelsService + .addService(ServiceDescriptorProto.newBuilder() + .setName("SignalChannelsService") + .addMethod(MethodDescriptorProto.newBuilder() + .setName("Publish") + .setInputType(".code.obp.grpc.signal.g1.PublishRequest") + .setOutputType(".code.obp.grpc.signal.g1.PublishResponse") + ) + .addMethod(MethodDescriptorProto.newBuilder() + .setName("Fetch") + .setInputType(".code.obp.grpc.signal.g1.FetchRequest") + .setOutputType(".code.obp.grpc.signal.g1.FetchResponse") + ) + .addMethod(MethodDescriptorProto.newBuilder() + .setName("ListChannels") + .setInputType(".code.obp.grpc.signal.g1.ListChannelsRequest") + .setOutputType(".code.obp.grpc.signal.g1.ListChannelsResponse") + ) + .addMethod(MethodDescriptorProto.newBuilder() + .setName("Subscribe") + .setInputType(".code.obp.grpc.signal.g1.SubscribeRequest") + .setOutputType(".code.obp.grpc.signal.g1.SignalMessage") + .setServerStreaming(true) + ) + ) + .build() + + com.google.protobuf.Descriptors.FileDescriptor.buildFrom( + fileProto, + Array(com.google.protobuf.TimestampProto.getDescriptor) + ) + } + + private def stringField(name: String, number: Int): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_STRING) + .setLabel(Label.LABEL_OPTIONAL) + + private def int32Field(name: String, number: Int): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_INT32) + .setLabel(Label.LABEL_OPTIONAL) + + private def int64Field(name: String, number: Int): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_INT64) + .setLabel(Label.LABEL_OPTIONAL) + + private def boolField(name: String, number: Int): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_BOOL) + .setLabel(Label.LABEL_OPTIONAL) + + private def messageField(name: String, number: Int, typeName: String): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_MESSAGE) + .setTypeName(typeName) + .setLabel(Label.LABEL_OPTIONAL) + + private def repeatedMessageField(name: String, number: Int, typeName: String): FieldDescriptorProto.Builder = + FieldDescriptorProto.newBuilder() + .setName(name).setNumber(number) + .setType(Type.TYPE_MESSAGE) + .setTypeName(typeName) + .setLabel(Label.LABEL_REPEATED) +} diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SubscribeRequest.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SubscribeRequest.scala new file mode 100644 index 0000000000..ebc4b9006c --- /dev/null +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SubscribeRequest.scala @@ -0,0 +1,95 @@ +// Hand-written to match the scalapb-generated shape used elsewhere in the +// gRPC layer (see chat/api and logcache/api). No protoc plugin is wired into +// the Maven build. Source of truth: obp-api/src/main/protobuf/signal.proto. +// Regenerate with scripts/gen_signal_grpc_messages.py if the proto changes. +// +// Protofile syntax: PROTO3 + +package code.obp.grpc.signal.api + +@SerialVersionUID(0L) +final case class SubscribeRequest( + channelName: _root_.scala.Predef.String = "" + ) extends scalapb.GeneratedMessage with scalapb.Message[SubscribeRequest] with scalapb.lenses.Updatable[SubscribeRequest] { + @transient + private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 + private[this] def __computeSerializedValue(): _root_.scala.Int = { + var __size = 0 + if (channelName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, channelName) } + __size + } + final override def serializedSize: _root_.scala.Int = { + var read = __serializedSizeCachedValue + if (read == 0) { + read = __computeSerializedValue() + __serializedSizeCachedValue = read + } + read + } + def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = { + { val __v = channelName; if (__v != "") _output__.writeString(1, __v) }; + } + def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.SubscribeRequest = { + var __channelName = this.channelName + var _done__ = false + while (!_done__) { + val _tag__ = _input__.readTag() + _tag__ match { + case 0 => _done__ = true + case 10 => + __channelName = _input__.readString() + case tag => _input__.skipField(tag) + } + } + code.obp.grpc.signal.api.SubscribeRequest( + channelName = __channelName + ) + } + def withChannelName(__v: _root_.scala.Predef.String): SubscribeRequest = copy(channelName = __v) + def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { + (__fieldNumber: @_root_.scala.unchecked) match { + case 1 => { + val __t = channelName + if (__t != "") __t else null + } + } + } + def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { + require(__field.containingMessage eq companion.scalaDescriptor) + (__field.number: @_root_.scala.unchecked) match { + case 1 => _root_.scalapb.descriptors.PString(channelName) + } + } + def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) + def companion = code.obp.grpc.signal.api.SubscribeRequest +} + +object SubscribeRequest extends scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SubscribeRequest] { + implicit def messageCompanion: scalapb.GeneratedMessageCompanion[code.obp.grpc.signal.api.SubscribeRequest] = this + def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): code.obp.grpc.signal.api.SubscribeRequest = { + require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.") + val __fields = javaDescriptor.getFields + code.obp.grpc.signal.api.SubscribeRequest( + __fieldsMap.getOrElse(__fields.get(0), "").asInstanceOf[_root_.scala.Predef.String] + ) + } + implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.SubscribeRequest] = _root_.scalapb.descriptors.Reads{ + case _root_.scalapb.descriptors.PMessage(__fieldsMap) => + require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.") + code.obp.grpc.signal.api.SubscribeRequest( + __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + ) + case _ => throw new RuntimeException("Expected PMessage") + } + def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = SignalProto.javaDescriptor.getMessageTypes.get(8) + def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = throw new UnsupportedOperationException("scalaDescriptor not available") + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty + def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) + lazy val defaultInstance = code.obp.grpc.signal.api.SubscribeRequest( + ) + implicit class SubscribeRequestLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, code.obp.grpc.signal.api.SubscribeRequest]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, code.obp.grpc.signal.api.SubscribeRequest](_l) { + def channelName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.channelName)((c_, f_) => c_.copy(channelName = f_)) + } + final val CHANNEL_NAME_FIELD_NUMBER = 1 +} diff --git a/obp-api/src/main/scala/code/signal/SignalChannels.scala b/obp-api/src/main/scala/code/signal/SignalChannels.scala new file mode 100644 index 0000000000..4607d7df5b --- /dev/null +++ b/obp-api/src/main/scala/code/signal/SignalChannels.scala @@ -0,0 +1,95 @@ +package code.signal + +import code.api.cache.RedisMessaging +import code.api.util.CustomJsonFormats +import code.api.v6_0_0.{PostSignalMessageJsonV600, SignalChannelInfoJsonV600, SignalMessageJsonV600, SignalMessagePublishedJsonV600, SignalMessagesJsonV600} +import com.openbankproject.commons.util.JsonAliases +import org.json4s.Extraction + +import java.util.UUID.randomUUID +import scala.util.Try + +/** + * The signal-channel behaviour shared by the REST endpoints (Http4s600) and + * the gRPC SignalChannelsService, so both transports build the same envelope, + * apply the same privacy filter and list channels the same way. Validation + * (channel name, size cap, dangerous characters) stays in the callers because + * each transport reports failures differently. + */ +object SignalChannels { + + private implicit val formats = CustomJsonFormats.formats + + private def utcTimestampNow(): String = { + val sdf = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") + sdf.setTimeZone(java.util.TimeZone.getTimeZone("UTC")) + sdf.format(new java.util.Date()) + } + + /** Build the envelope, store it and notify live subscribers. */ + def publish(channelName: String, senderUserId: String, senderConsumerId: String, post: PostSignalMessageJsonV600): SignalMessagePublishedJsonV600 = { + val messageId = randomUUID().toString + val timestamp = utcTimestampNow() + val envelope = SignalMessageJsonV600( + message_id = messageId, channel_name = channelName, + sender_consumer_id = senderConsumerId, sender_user_id = senderUserId, + to_user_id = post.to_user_id, timestamp = timestamp, + message_type = post.message_type.getOrElse(""), + payload = post.payload) + // The sequence is stamped inside Redis (atomically with the push); strip the placeholder. + val msgStr = JsonAliases.compactRender(Extraction.decompose(envelope).removeField(_._1 == "sequence")) + val (sequence, count) = RedisMessaging.publishMessage(channelName, msgStr) + SignalMessagePublishedJsonV600(messageId, channelName, timestamp, count, sequence) + } + + def parseMessage(raw: String): Option[SignalMessageJsonV600] = + Try(JsonAliases.parse(raw).extract[SignalMessageJsonV600]).toOption + + /** Broadcasts are visible to everyone; a private message only to its sender and recipient. */ + def isVisibleTo(msg: SignalMessageJsonV600, userId: String): Boolean = + msg.to_user_id.isEmpty || msg.to_user_id.contains(userId) || msg.sender_user_id == userId + + /** + * Fetch a page of a channel and apply the privacy filter for `userId`. + * + * With `afterSequence` this is a cursor read (messages newer than that sequence), which is the + * way to poll: offset paging drifts once the channel is trimmed to its newest N messages. + * Without it, plain offset/limit paging over whatever the channel currently holds. + * In both modes next_after_sequence is the sequence of the last raw message in the window, + * visible or not, so a caller can always advance. + */ + def fetch(channelName: String, offset: Int, limit: Int, afterSequence: Option[Long], userId: String): SignalMessagesJsonV600 = + afterSequence match { + case Some(after) => + val (raw, total, latest) = RedisMessaging.fetchMessagesAfter(channelName, after, limit) + val nextAfter = raw.lastOption.map(RedisMessaging.sequenceOf).getOrElse(after) + page(channelName, raw, total, hasMore = nextAfter < latest, latest, nextAfter, userId) + case None => + val (raw, total) = RedisMessaging.fetchMessages(channelName, offset, limit) + val latest = RedisMessaging.latestSequence(channelName) + val nextAfter = raw.lastOption.map(RedisMessaging.sequenceOf).getOrElse(0L) + page(channelName, raw, total, hasMore = (offset + limit) < total, latest, nextAfter, userId) + } + + private def page(channelName: String, raw: List[String], total: Long, hasMore: Boolean, + latest: Long, nextAfter: Long, userId: String): SignalMessagesJsonV600 = { + val visible = raw.flatMap(parseMessage).filter(isVisibleTo(_, userId)) + SignalMessagesJsonV600(channelName, visible, total, hasMore, latest, nextAfter) + } + + /** Channels holding at least one broadcast message. Private-only channels are not listed. */ + def listBroadcastChannels(): List[SignalChannelInfoJsonV600] = + RedisMessaging.listChannels().flatMap { name => + RedisMessaging.channelInfo(name).flatMap { case (count, ttl) => + val (messages, _) = RedisMessaging.fetchMessages(name, 0, count.toInt) + val hasBroadcast = messages.exists(s => parseMessage(s).exists(_.to_user_id.isEmpty)) + if (hasBroadcast) Some(SignalChannelInfoJsonV600(name, count, ttl)) else None + } + } + + /** Every channel, private-only ones included. */ + def listAllChannels(): List[SignalChannelInfoJsonV600] = + RedisMessaging.listChannels().flatMap { name => + RedisMessaging.channelInfo(name).map { case (count, ttl) => SignalChannelInfoJsonV600(name, count, ttl) } + } +} diff --git a/obp-api/src/main/scala/code/signal/SignalEventBus.scala b/obp-api/src/main/scala/code/signal/SignalEventBus.scala new file mode 100644 index 0000000000..a3b10207c3 --- /dev/null +++ b/obp-api/src/main/scala/code/signal/SignalEventBus.scala @@ -0,0 +1,111 @@ +package code.signal + +import code.api.cache.Redis +import code.util.Helper.MdcLoggable +import io.grpc.stub.StreamObserver +import redis.clients.jedis.{Jedis, JedisPubSub} + +import java.util.concurrent.{ConcurrentHashMap, CopyOnWriteArrayList} +import scala.jdk.CollectionConverters._ + +/** + * Redis pub/sub fan-out for signal channels: the live half of the design in + * the Signal Channels glossary entry. Every publish (REST or gRPC) goes + * through RedisMessaging.publishMessage, which stores the envelope in the + * channel list and then PUBLISHes it on `obp_signal:`. This bus + * holds one pattern subscription on `obp_signal:*` for the process and hands + * each envelope to the gRPC Subscribe streams registered for that channel. + * + * Nothing is buffered: a subscriber that connects after a message was + * published never sees it. Late joiners ask the other agents, or Fetch. + * + * Same shape and lifecycle contract as ChatEventBus: start() is a no-op once + * running, and only the ObpGrpcServer instance that started it stops it. + */ +object SignalEventBus extends MdcLoggable { + + private val CHANNEL_PREFIX = "obp_signal:" + + /** The Redis pub/sub channel that carries live envelopes for one signal channel. */ + def redisChannel(channelName: String): String = CHANNEL_PREFIX + channelName + + // channel name -> gRPC bridges waiting on it + private val observers = new ConcurrentHashMap[String, CopyOnWriteArrayList[StreamObserver[String]]]() + + @volatile private var subscriberThread: Thread = _ + @volatile private var subscriberJedis: Jedis = _ + @volatile private var pubSub: JedisPubSub = _ + @volatile private var running = false + + def subscribe(channelName: String, observer: StreamObserver[String]): Unit = { + observers.computeIfAbsent(channelName, _ => new CopyOnWriteArrayList[StreamObserver[String]]()) + observers.get(channelName).add(observer) + logger.info(s"SignalEventBus says: Observer subscribed to $channelName (total: ${observers.get(channelName).size})") + } + + def unsubscribe(channelName: String, observer: StreamObserver[String]): Unit = { + val list = observers.get(channelName) + if (list != null) { + list.remove(observer) + logger.info(s"SignalEventBus says: Observer unsubscribed from $channelName (remaining: ${list.size})") + if (list.isEmpty) observers.remove(channelName) + } + } + + /** How many gRPC streams are currently attached to a channel on this instance. */ + def subscriberCount(channelName: String): Int = + Option(observers.get(channelName)).map(_.size).getOrElse(0) + + def start(): Unit = { + if (running) return + running = true + + pubSub = new JedisPubSub { + override def onPMessage(pattern: String, channel: String, message: String): Unit = { + val channelName = channel.stripPrefix(CHANNEL_PREFIX) + val list = observers.get(channelName) + if (list != null) { + list.asScala.foreach { observer => + try { + observer.synchronized { + observer.onNext(message) + } + } catch { + case e: Throwable => + logger.warn(s"SignalEventBus says: Failed to deliver on $channelName, removing observer: ${e.getMessage}") + list.remove(observer) + } + } + } + } + } + + subscriberThread = new Thread(() => { + try { + // Dedicated connection: a subscribing Jedis cannot be returned to the pool. + subscriberJedis = Redis.newSubscriberConnection() + logger.info(s"SignalEventBus says: Redis subscriber started, pattern-subscribing to ${CHANNEL_PREFIX}*") + subscriberJedis.psubscribe(pubSub, s"${CHANNEL_PREFIX}*") + } catch { + case e: Throwable if running => + logger.error(s"SignalEventBus says: Redis subscriber thread died: ${e.getMessage}") + case _: Throwable => // shutting down, ignore + } + }, "signal-event-bus-subscriber") + subscriberThread.setDaemon(true) + subscriberThread.start() + + logger.info("SignalEventBus says: Started") + } + + /** Whether this bus is already subscribed, so a caller can tell whether it started it. */ + def isRunning: Boolean = running + + def stop(): Unit = { + running = false + try { if (pubSub != null) pubSub.punsubscribe() } catch { case _: Throwable => } + try { if (subscriberJedis != null) subscriberJedis.close() } catch { case _: Throwable => } + observers.clear() + logger.info("SignalEventBus says: Stopped") + } +} diff --git a/obp-api/src/main/scala/code/users/UserReference.scala b/obp-api/src/main/scala/code/users/UserReference.scala index 84dd382491..04c89455ba 100644 --- a/obp-api/src/main/scala/code/users/UserReference.scala +++ b/obp-api/src/main/scala/code/users/UserReference.scala @@ -80,6 +80,8 @@ object UserReference { case object AccountAccessRequestRequestor extends UserReference(UseOnBehalfOfUserId, "code.accountaccessrequest.AccountAccessRequest", List("RequestorUserId")) case object AccountAccessRequestTarget extends UserReference(UseOnBehalfOfUserId, "code.accountaccessrequest.AccountAccessRequest", List("TargetUserId"), "explicit target: a consent user named here is rejected at the endpoint") case object AccountAccessRequestChecker extends UserReference(UseOnBehalfOfUserId, "code.accountaccessrequest.AccountAccessRequest", List("CheckerUserId")) + case object DynamicChangeRequestRequestor extends UserReference(UseOnBehalfOfUserId, "code.dynamicchangerequest.DynamicChangeRequest", List("RequestorUserId"), "maker of a dynamic-code change") + case object DynamicChangeRequestChecker extends UserReference(UseOnBehalfOfUserId, "code.dynamicchangerequest.DynamicChangeRequest", List("CheckerUserId"), "checker; must differ from the requestor") case object EntitlementRequestUser extends UserReference(UseOnBehalfOfUserId, "code.entitlementrequest.MappedEntitlementRequest", List("mUserId")) case object UserScopeUser extends UserReference(UseOnBehalfOfUserId, "code.scope.MappedUserScope", List("mUserId")) case object ApiCollectionUser extends UserReference(UseOnBehalfOfUserId, "code.apicollection.ApiCollection", List("UserId")) @@ -157,6 +159,8 @@ object UserReference { AccountAccessRequestRequestor, AccountAccessRequestTarget, AccountAccessRequestChecker, + DynamicChangeRequestRequestor, + DynamicChangeRequestChecker, EntitlementRequestUser, UserScopeUser, ApiCollectionUser, @@ -214,6 +218,7 @@ object UserReference { ("code.model.dataAccess.MappedBankAccount", "holder", "free-text holder name"), ("code.transaction.MappedTransaction", "counterpartyAccountHolder", "free-text name"), ("code.accountaccessrequest.AccountAccessRequest", "CheckerComment", "text"), + ("code.dynamicchangerequest.DynamicChangeRequest", "CheckerComment", "text"), ("code.kycchecks.MappedKycCheck", "mStaffName", "text"), ("code.meetings.MappedMeeting", "mStaffToken", "token"), ("code.entitlement.MappedEntitlement", "mCreatedByProcess", "process tag"), diff --git a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala index 13c56f0fdc..a3aa26ba2c 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala @@ -7,6 +7,7 @@ import code.api.util.ErrorMessages.{SignalMessageContainsDangerousCharacters, Si import code.signal.SignalContentPolicy import com.openbankproject.commons.model.ErrorMessage import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST.JValue import org.scalatest.Tag /** @@ -15,13 +16,19 @@ import org.scalatest.Tag * Every scenario here fails BEFORE RedisMessaging is touched (auth and role * checks run in the middleware; the size and character checks run in the * handler ahead of the Redis publish), so no Redis instance is needed. - * Success paths (publish 201, delete 200) require Redis and are not covered. + * The cursor scenario needs a reachable Redis and is cancelled, not failed, without one. */ class SignalChannelTest extends V600ServerSetup { object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) object ApiEndpointPublish extends Tag("publishSignalMessage") object ApiEndpointDelete extends Tag("deleteSignalChannel") + object ApiEndpointGetMessages extends Tag("getSignalMessages") + + private def redisReachable: Boolean = scala.util.Try { + val jedis = code.api.cache.Redis.jedisPool.getResource + try jedis.ping() finally jedis.close() + }.isSuccess private def publishRequest = (v6_0_0_Request / "signal" / "channels" / "test-channel" / "messages").POST private def deleteRequest = (v6_0_0_Request / "signal" / "channels" / "test-channel").DELETE @@ -77,6 +84,45 @@ class SignalChannelTest extends V600ServerSetup { } } + feature(s"Get Signal Messages - GET /obp/v6.0.0/signal/channels/CHANNEL_NAME/messages - $VersionOfApi") { + + scenario("a non-numeric after_sequence should fail with 400 OBP-10002", ApiEndpointGetMessages, VersionOfApi) { + val request = (v6_0_0_Request / "signal" / "channels" / "test-channel" / "messages").GET <@ (user1) < 0L + makePostRequest(publish, """{"payload":{"n":2}}""").code should equal(201) + makePostRequest(publish, """{"payload":{"n":3}}""").code should equal(201) + + val read = (v6_0_0_Request / "signal" / "channels" / channelName / "messages").GET <@ (user1) + val newer = makeGetRequest(read < firstSeq) should equal(true) + (newer.body \ "has_more").extract[Boolean] should equal(false) + val nextAfter = (newer.body \ "next_after_sequence").extract[Long] + nextAfter should equal((newer.body \ "latest_sequence").extract[Long]) + + val nothingNew = makeGetRequest(read < "true") + + private def enableMakerChecker(): Unit = { + dynamicCodeOn() + setPropsValues("dynamic_code_requires_approval" -> "true") + } + + private def grant(userId: String, roles: ApiRole*): Unit = + roles.foreach(r => Entitlement.entitlement.vend.addEntitlement("", userId, r.toString)) + + private def makerRoles(): Unit = + grant(resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc, ApiRole.canUpdateDynamicResourceDoc, ApiRole.canDeleteDynamicResourceDoc) + + private def checkerRoles(): Unit = + grant(resourceUser2.userId, ApiRole.canApproveDynamicChangeRequest, ApiRole.canGetDynamicChangeRequests) + + private def newDoc(suffix: String) = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = s"makerChecker$suffix", + requestUrl = s"/mc_native_user_$suffix/MY_USER_ID" + ) + + private def storedDoc(url: String): Option[DynamicResourceDoc] = + DynamicResourceDoc.find(By(DynamicResourceDoc.RequestUrl, url), By(DynamicResourceDoc.RequestVerb, "POST")).toOption + + private def callDynamicEndpoint(suffix: String) = { + val req = (dynamicEndpoint / "dynamic-resource-doc" / s"mc_native_user_$suffix" / "user-xyz").POST <@ (user1) + makePostRequest(req, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + } + + private def str(json: org.json4s.JValue, field: String): String = (json \ field).values.toString + + feature("Maker/checker disabled: today's behaviour is unchanged") { + scenario("a v4 create is applied directly, the row is active with no approved hash, and the endpoint runs", VersionOfApi) { + dynamicCodeOn(); makerRoles() + val doc = newDoc("off") + val resp = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + resp.codeIs(201) + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not stored")) + row.IsActive.get should be(true) + Option(row.ApprovedHash.get).getOrElse("") should be("") + callDynamicEndpoint("off").codeIs(200) + } + } + + feature("Maker/checker enabled: v4 writes are queued and applied only after a second user approves") { + + scenario("create is intercepted (202), nothing is stored, and the same user cannot approve", ApiEndpoint3, ApiEndpoint4, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + grant(resourceUser1.userId, ApiRole.canApproveDynamicChangeRequest, ApiRole.canGetDynamicChangeRequests) + val doc = newDoc("same") + + When("the maker POSTs the dynamic resource doc via v4.0.0") + val resp = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + Then("the write is queued, not applied") + resp.codeIs(202) + str(resp.body, "status") should equal("INITIATED") + str(resp.body, "target_type") should equal("DYNAMIC_RESOURCE_DOC") + str(resp.body, "operation") should equal("CREATE") + str(resp.body, "requestor_user_id") should equal(resourceUser1.userId) + str(resp.body, "request_path") should include("/management/dynamic-resource-docs") + val hash = str(resp.body, "payload_hash") + hash.length should equal(64) + hash should equal(MakerChecker.payloadHash(write(doc))) + storedDoc(doc.requestUrl) should be(None) + callDynamicEndpoint("same").codeIs(404) + + val id = str(resp.body, "dynamic_change_request_id") + When("the maker reads it back") + val get = makeGetRequest((v7 / "management" / "dynamic-change-requests" / id).GET <@ (user1)) + get.codeIs(200) + str(get.body, "payload_hash") should equal(hash) + + When("the same user tries to approve") + val approve = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user1), + s"""{"payload_hash":"$hash","checker_comment":"self"}""") + Then("maker/checker separation is enforced") + approve.codeIs(400) + approve.body.toString should include(MakerCheckerSameUser.split(":").head) + storedDoc(doc.requestUrl) should be(None) + } + + scenario("a second user approves by hash: wrong hash fails, right hash applies, the endpoint runs, a second approval fails", ApiEndpoint4, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val doc = newDoc("ok") + val resp = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + resp.codeIs(202) + val id = str(resp.body, "dynamic_change_request_id") + val hash = str(resp.body, "payload_hash") + + When("the checker sends a different hash") + val bad = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user2), + """{"payload_hash":"0000000000000000000000000000000000000000000000000000000000000000"}""") + bad.codeIs(400) + bad.body.toString should include(DynamicChangeRequestHashMismatch.split(":").head) + storedDoc(doc.requestUrl) should be(None) + + When("the checker approves the exact hash (sha256: prefix accepted)") + val ok = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user2), + s"""{"payload_hash":"sha256:$hash","checker_comment":"reviewed"}""") + Then("the request is APPROVED and the doc exists with its body hash approved") + ok.codeIs(200) + str(ok.body, "status") should equal("APPROVED") + str(ok.body, "checker_user_id") should equal(resourceUser2.userId) + str(ok.body, "checker_comment") should equal("reviewed") + (ok.body \ "target_id").values.toString.nonEmpty should be(true) + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) + row.CreatedByUserId.get should be(resourceUser1.userId) + row.MethodBodyHash.get should be(APIUtil.sha256Hex(URLDecoder.decode(doc.methodBody, "UTF-8"))) + row.ApprovedHash.get should be(row.MethodBodyHash.get) + row.IsActive.get should be(true) + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(true) + + Then("the compiled endpoint is served") + val call = callDynamicEndpoint("ok") + call.codeIs(200) + call.body.toString should include("user-xyz_from_path") + + When("someone approves again") + val again = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user2), + s"""{"payload_hash":"$hash"}""") + again.codeIs(400) + again.body.toString should include(DynamicChangeRequestNotInitiated.split(":").head) + + Then("the v7 provenance view shows the approval") + grant(resourceUser2.userId, ApiRole.canGetDynamicResourceDoc) + val prov = makeGetRequest((v7 / "management" / "dynamic-resource-docs" / row.DynamicResourceDocId.get).GET <@ (user2)) + prov.codeIs(200) + str(prov.body \ "provenance", "approved_hash") should equal(row.ApprovedHash.get) + (prov.body \ "provenance" \ "is_active").values should equal(true) + } + + scenario("the execution guard holds against direct database edits and deactivation is a single-approver action", ApiEndpoint8, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val doc = newDoc("guard") + val resp = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + val id = str(resp.body, "dynamic_change_request_id") + makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(resp.body, "payload_hash")}"}""").codeIs(200) + callDynamicEndpoint("guard").codeIs(200) + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) + + When("the body hash is changed behind the API's back") + row.MethodBodyHash("tampered").save + Then("the endpoint is no longer served") + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(false) + callDynamicEndpoint("guard").codeIs(404) + row.MethodBodyHash(row.ApprovedHash.get).save + callDynamicEndpoint("guard").codeIs(200) + + When("a maker without the approver role tries to deactivate") + val forbidden = makePostRequest((v7 / "management" / "dynamic-resource-docs" / row.DynamicResourceDocId.get / "deactivation").POST <@ (user1), """{"comment":"x"}""") + forbidden.codeIs(403) + + When("the approver deactivates directly") + val off = makePostRequest((v7 / "management" / "dynamic-resource-docs" / row.DynamicResourceDocId.get / "deactivation").POST <@ (user2), """{"comment":"suspected leak"}""") + Then("it is audited as an APPROVED DEACTIVATE row and the endpoint stops") + off.codeIs(200) + str(off.body, "operation") should equal("DEACTIVATE") + str(off.body, "status") should equal("APPROVED") + str(off.body, "requestor_user_id") should equal(resourceUser2.userId) + storedDoc(doc.requestUrl).get.IsActive.get should be(false) + callDynamicEndpoint("guard").codeIs(404) + + When("the maker asks to re-activate via an explicit change request and the approver approves it") + val activate = makePostRequest((v7 / "management" / "dynamic-change-requests").POST <@ (user1), + s"""{"target_type":"DYNAMIC_RESOURCE_DOC","operation":"ACTIVATE","target_id":"${row.DynamicResourceDocId.get}","proposed_payload":{},"business_justification":"reviewed, false alarm"}""") + activate.codeIs(201) + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(activate.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(activate.body, "payload_hash")}"}""").codeIs(200) + storedDoc(doc.requestUrl).get.IsActive.get should be(true) + callDynamicEndpoint("guard").codeIs(200) + } + + scenario("seeding the approved hash of pre-existing rows runs once per database, not at every boot", VersionOfApi) { + makerRoles(); checkerRoles() + val logProvider = code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend + code.migration.MigrationScriptLog.findAll(By(code.migration.MigrationScriptLog.Name, MakerChecker.seedMigrationName)).foreach(_.delete_!) + def approvedHashOf(url: String): String = Option(storedDoc(url).getOrElse(fail("doc not created")).ApprovedHash.get).getOrElse("") + + Given("a row created before approval was required, so it has no approved hash") + dynamicCodeOn(); setPropsValues("dynamic_code_requires_approval" -> "false") + val legacy = newDoc("legacy") + makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(legacy)).codeIs(201) + approvedHashOf(legacy.requestUrl) should equal("") + setPropsValues("dynamic_code_requires_approval" -> "true") + callDynamicEndpoint("legacy").codeIs(404) + + When("the instance boots with approval required for the first time") + MakerChecker.seedApprovedHashesIfEnabled() + Then("the row's current body is treated as approved and the seed is logged") + approvedHashOf(legacy.requestUrl) should equal(APIUtil.sha256Hex(URLDecoder.decode(legacy.methodBody, "UTF-8"))) + callDynamicEndpoint("legacy").codeIs(200) + logProvider.isExecuted(MakerChecker.seedMigrationName) should be(true) + + When("a row with no approved hash appears after the seed, e.g. inserted while approval was switched off") + setPropsValues("dynamic_code_requires_approval" -> "false") + val late = newDoc("late") + makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(late)).codeIs(201) + setPropsValues("dynamic_code_requires_approval" -> "true") + MakerChecker.seedApprovedHashesIfEnabled() + Then("a later boot does not bless it: it stays unexecutable until a checker approves it") + approvedHashOf(late.requestUrl) should equal("") + MakerChecker.isExecutableDynamicResourceDoc(storedDoc(late.requestUrl).get.DynamicResourceDocId.get) should be(false) + callDynamicEndpoint("late").codeIs(404) + } + + scenario("an update is queued with the live hash; rejection needs a comment and leaves the target untouched", ApiEndpoint5, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val doc = newDoc("upd") + val created = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(created.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(created.body, "payload_hash")}"}""").codeIs(200) + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) + val docId = row.DynamicResourceDocId.get + + When("the maker PUTs a changed body") + val changed = doc.copy(dynamicResourceDocId = Some(docId), summary = "changed summary") + val put = makePutRequest((v4 / "management" / "dynamic-resource-docs" / docId).PUT <@ (user1), write(changed)) + put.codeIs(202) + str(put.body, "operation") should equal("UPDATE") + str(put.body, "target_id") should equal(docId) + str(put.body, "current_payload_hash") should equal(row.MethodBodyHash.get) + (put.body \ "current_payload" \ "summary").values.toString should equal(doc.summary) + (put.body \ "proposed_payload" \ "summary").values.toString should equal("changed summary") + storedDoc(doc.requestUrl).get.Summary.get should equal(doc.summary) + val id = str(put.body, "dynamic_change_request_id") + + When("the checker rejects without a comment") + makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "rejection").POST <@ (user2), """{"comment":" "}""").codeIs(400) + When("the checker rejects with a comment") + val rej = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "rejection").POST <@ (user2), """{"comment":"not now"}""") + rej.codeIs(200) + str(rej.body, "status") should equal("REJECTED") + str(rej.body, "checker_comment") should equal("not now") + storedDoc(doc.requestUrl).get.Summary.get should equal(doc.summary) + + When("a delete is requested and approved") + val del = makeDeleteRequest((v4 / "management" / "dynamic-resource-docs" / docId).DELETE <@ (user1)) + del.codeIs(202) + str(del.body, "operation") should equal("DELETE") + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(del.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(del.body, "payload_hash")}"}""").codeIs(200) + storedDoc(doc.requestUrl) should be(None) + } + + scenario("only the requestor can withdraw; listings and /my work; roles and auth are enforced", ApiEndpoint1, ApiEndpoint2, ApiEndpoint6, ApiEndpoint7, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val doc = newDoc("wd") + val resp = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + val id = str(resp.body, "dynamic_change_request_id") + + When("another user tries to withdraw") + val other = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "withdrawal").POST <@ (user2), """{"comment":"mine now"}""") + other.codeIs(400) + other.body.toString should include(DynamicChangeRequestNotRequestor.split(":").head) + When("the requestor withdraws") + val wd = makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "withdrawal").POST <@ (user1), """{"comment":"superseded"}""") + wd.codeIs(200) + str(wd.body, "status") should equal("WITHDRAWN") + + Then("/my lists it for the maker and the management listing filters by status") + val mine = makeGetRequest((v7 / "my" / "dynamic-change-requests").GET <@ (user1)) + mine.codeIs(200) + (mine.body \ "dynamic_change_requests").asInstanceOf[JArray].arr.exists(r => str(r, "dynamic_change_request_id") == id) should be(true) + val listed = makeGetRequest((v7 / "management" / "dynamic-change-requests").GET <@ (user2) < "WITHDRAWN")) + listed.codeIs(200) + (listed.body \ "dynamic_change_requests").asInstanceOf[JArray].arr.forall(r => str(r, "status") == "WITHDRAWN") should be(true) + (listed.body \ "dynamic_change_requests").asInstanceOf[JArray].arr.exists(r => str(r, "dynamic_change_request_id") == id) should be(true) + + Then("the listing needs the role and authentication") + makeGetRequest((v7 / "management" / "dynamic-change-requests").GET <@ (user3)).codeIs(403) + makeGetRequest((v7 / "management" / "dynamic-change-requests").GET).codeIs(401) + makePostRequest((v7 / "management" / "dynamic-change-requests" / id / "approval").POST <@ (user3), """{"payload_hash":"x"}""").codeIs(403) + + When("an explicit submission is made without the maker role for that target type") + val noRole = makePostRequest((v7 / "management" / "dynamic-change-requests").POST <@ (user3), + s"""{"target_type":"DYNAMIC_RESOURCE_DOC","operation":"CREATE","proposed_payload":${write(newDoc("explicit"))},"business_justification":"j"}""") + noRole.codeIs(403) + When("an explicit submission is made by the maker and approved") + val explicitDoc = newDoc("explicit") + val explicit = makePostRequest((v7 / "management" / "dynamic-change-requests").POST <@ (user1), + s"""{"target_type":"DYNAMIC_RESOURCE_DOC","operation":"CREATE","proposed_payload":${write(explicitDoc)},"business_justification":"needed by mobile"}""") + explicit.codeIs(201) + str(explicit.body, "business_justification") should equal("needed by mobile") + str(explicit.body, "request_path") should equal("/obp/v4.0.0/management/dynamic-resource-docs") + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(explicit.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(explicit.body, "payload_hash")}"}""").codeIs(200) + storedDoc(explicitDoc.requestUrl).isDefined should be(true) + callDynamicEndpoint("explicit").codeIs(200) + } + } +} diff --git a/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala b/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala new file mode 100644 index 0000000000..1f471c1f1a --- /dev/null +++ b/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala @@ -0,0 +1,232 @@ +package code.obp.grpc + +import code.api.cache.Redis +import code.api.util.ErrorMessages.{InvalidJsonFormat, InvalidSignalChannelName, SignalMessageContainsDangerousCharacters, SignalMessageTooLong} +import code.obp.grpc.signal.api._ +import code.setup.ServerSetupWithTestData +import code.signal.{SignalContentPolicy, SignalEventBus} +import io.grpc.stub.{MetadataUtils, StreamObserver} +import io.grpc.{ManagedChannel, ManagedChannelBuilder, Metadata, Status, StatusRuntimeException} +import org.scalatest.Tag + +import java.util.concurrent.{CountDownLatch, TimeUnit} +import scala.util.Try + +/** + * SignalChannelsService over a real socket, following ObpGrpcServerSmokeTest. + * + * The validation scenarios fail before Redis is touched (same as the REST + * SignalChannelTest), so they run anywhere. The round-trip scenarios need a + * reachable Redis and are cancelled, not failed, without one. + */ +class SignalChannelsGrpcTest extends ServerSetupWithTestData { + + object GrpcSignal extends Tag("GrpcSignal") + + private var grpcServer: ObpGrpcServer = _ + private var channel: ManagedChannel = _ + + override def beforeAll(): Unit = { + super.beforeAll() + grpcServer = new ObpGrpcServer(scala.concurrent.ExecutionContext.global, port = 0) + grpcServer.start() + channel = ManagedChannelBuilder + .forAddress("localhost", grpcServer.boundPort) + .usePlaintext() + .asInstanceOf[ManagedChannelBuilder[_]] + .build() + } + + override def afterAll(): Unit = { + if (channel != null) channel.shutdownNow() + if (grpcServer != null) grpcServer.stop() + super.afterAll() + } + + private def authMetadata(token: String): Metadata = { + val metadata = new Metadata() + metadata.put(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER), s"""DirectLogin token="$token"""") + metadata + } + + private def tokenOf(user: Option[(_, code.api.util.APIUtil.OAuth.Token)]): String = + user.map(_._2.value).getOrElse(fail("no DirectLogin token")) + + private def blockingStub(token: String): SignalChannelsServiceGrpc.SignalChannelsServiceBlockingStub = + SignalChannelsServiceGrpc.blockingStub(channel).withInterceptors(MetadataUtils.newAttachHeadersInterceptor(authMetadata(token))) + + private def asyncStub(token: String): SignalChannelsServiceGrpc.SignalChannelsServiceStub = + SignalChannelsServiceGrpc.stub(channel).withInterceptors(MetadataUtils.newAttachHeadersInterceptor(authMetadata(token))) + + private def statusOf(body: => Any): Status = intercept[StatusRuntimeException](body).getStatus + + private def redisReachable: Boolean = Try { + val jedis = Redis.jedisPool.getResource + try jedis.ping() finally jedis.close() + }.isSuccess + + private val cleanPayload = """{"message":"Please report what time it is where you are"}""" + + feature("SignalChannelsService validation matches the REST endpoints") { + + scenario("a call with no credentials is rejected", GrpcSignal) { + val status = statusOf(SignalChannelsServiceGrpc.blockingStub(channel).publish(PublishRequest("test-channel", "", "", cleanPayload))) + status.getCode should equal(Status.Code.UNAUTHENTICATED) + } + + scenario("an invalid channel name is INVALID_ARGUMENT with the REST error message", GrpcSignal) { + val status = statusOf(blockingStub(tokenOf(user1)).publish(PublishRequest("bad channel name!", "", "", cleanPayload))) + status.getCode should equal(Status.Code.INVALID_ARGUMENT) + status.getDescription should startWith(InvalidSignalChannelName) + } + + scenario("a payload that is not JSON is INVALID_ARGUMENT", GrpcSignal) { + val status = statusOf(blockingStub(tokenOf(user1)).publish(PublishRequest("test-channel", "", "", "not json"))) + status.getCode should equal(Status.Code.INVALID_ARGUMENT) + status.getDescription should startWith(InvalidJsonFormat) + } + + scenario("a payload over the size cap is INVALID_ARGUMENT OBP-39019", GrpcSignal) { + val oversized = "x" * (SignalContentPolicy.maxPayloadLength + 1) + val status = statusOf(blockingStub(tokenOf(user1)).publish(PublishRequest("test-channel", "", "", s"""{"data":"$oversized"}"""))) + status.getCode should equal(Status.Code.INVALID_ARGUMENT) + status.getDescription should startWith(SignalMessageTooLong) + } + + scenario("a payload containing a bidi override character is INVALID_ARGUMENT OBP-39020", GrpcSignal) { + // Backslash-u escape on the wire; parses to the RLO code point, as in SignalChannelTest. + val status = statusOf(blockingStub(tokenOf(user1)).publish(PublishRequest("test-channel", "", "", "{\"note\":\"click\\u202ehere\"}"))) + status.getCode should equal(Status.Code.INVALID_ARGUMENT) + status.getDescription should equal(SignalMessageContainsDangerousCharacters) + } + + scenario("a message_type containing a control character is INVALID_ARGUMENT OBP-39020", GrpcSignal) { + val withNul = "te" + 0.toChar + "xt" + val status = statusOf(blockingStub(tokenOf(user1)).publish(PublishRequest("test-channel", "", withNul, cleanPayload))) + status.getCode should equal(Status.Code.INVALID_ARGUMENT) + status.getDescription should equal(SignalMessageContainsDangerousCharacters) + } + + scenario("Fetch and Subscribe reject an invalid channel name before touching Redis", GrpcSignal) { + statusOf(blockingStub(tokenOf(user1)).fetch(FetchRequest("a" * 129, 0, 10))).getCode should equal(Status.Code.INVALID_ARGUMENT) + // The blocking iterator only fails when it is first read. + statusOf(blockingStub(tokenOf(user1)).subscribe(SubscribeRequest("a" * 129)).hasNext).getCode should equal(Status.Code.INVALID_ARGUMENT) + } + } + + feature("SignalChannelsService reads and writes the same Redis storage as REST") { + + scenario("Publish, Fetch and ListChannels round-trip a broadcast, and a private message stays private", GrpcSignal) { + if (!redisReachable) cancel("Redis is not reachable from this test JVM") + val channelName = s"grpc-test-${java.util.UUID.randomUUID().toString.take(8)}" + val publisher = blockingStub(tokenOf(user1)) + val publisherUserId = resourceUser1.userId + val otherUserId = resourceUser2.userId + + val broadcast = publisher.publish(PublishRequest(channelName, "", "task-request", cleanPayload)) + broadcast.channelName should equal(channelName) + broadcast.messageId should not be empty + broadcast.channelMessageCount should equal(1L) + broadcast.timestamp.isDefined should equal(true) + + val privateMsg = publisher.publish(PublishRequest(channelName, otherUserId, "private", """{"secret":true}""")) + privateMsg.channelMessageCount should equal(2L) + // A private message to a third party is invisible to everyone but sender and recipient. + publisher.publish(PublishRequest(channelName, "someone-else", "private", """{"secret":true}""")).channelMessageCount should equal(3L) + + val seenBySender = publisher.fetch(FetchRequest(channelName, 0, 10)) + seenBySender.totalCount should equal(3L) + // Sequences are stamped, strictly increasing, and reported consistently. + broadcast.sequence should be > 0L + privateMsg.sequence should be > broadcast.sequence + seenBySender.messages.map(_.sequence) should equal(seenBySender.messages.map(_.sequence).sorted) + seenBySender.latestSequence should be > privateMsg.sequence + seenBySender.nextAfterSequence should equal(seenBySender.latestSequence) + // Cursor read: only what came after the broadcast. + val afterBroadcast = publisher.fetch(FetchRequest(channelName, 0, 10, broadcast.sequence)) + afterBroadcast.messages.map(_.messageId) should equal(Seq(privateMsg.messageId, seenBySender.messages.last.messageId)) + afterBroadcast.hasMore should equal(false) + publisher.fetch(FetchRequest(channelName, 0, 10, seenBySender.latestSequence)).messages shouldBe empty + // The stranger's cursor advances past private messages it cannot see. + val strangerAfter = blockingStub(tokenOf(user3)).fetch(FetchRequest(channelName, 0, 10, broadcast.sequence)) + strangerAfter.messages shouldBe empty + strangerAfter.nextAfterSequence should equal(seenBySender.latestSequence) + seenBySender.messages.map(_.messageId) should contain allOf (broadcast.messageId, privateMsg.messageId) + seenBySender.messages.size should equal(3) + val first = seenBySender.messages.find(_.messageId == broadcast.messageId).get + first.payloadJson should equal(cleanPayload) + first.messageType should equal("task-request") + first.senderUserId should equal(publisherUserId) + first.toUserId should equal("") + + val seenByRecipient = blockingStub(tokenOf(user2)).fetch(FetchRequest(channelName, 0, 10)) + seenByRecipient.messages.map(_.messageId).toSet should equal(Set(broadcast.messageId, privateMsg.messageId)) + + val seenByStranger = blockingStub(tokenOf(user3)).fetch(FetchRequest(channelName, 0, 10)) + seenByStranger.messages.map(_.messageId) should equal(Seq(broadcast.messageId)) + + publisher.listChannels(ListChannelsRequest()).channels.map(_.channelName) should contain(channelName) + + code.api.cache.RedisMessaging.deleteChannel(channelName) + } + + scenario("a cursor read survives the channel being trimmed, where an offset read skips messages", GrpcSignal) { + if (!redisReachable) cancel("Redis is not reachable from this test JVM") + val channelName = s"grpc-trim-${java.util.UUID.randomUUID().toString.take(8)}" + val reader = blockingStub(tokenOf(user1)) + def envelope(n: Int) = + s"""{"message_id":"m$n","channel_name":"$channelName","sender_consumer_id":"","sender_user_id":"${resourceUser1.userId}","timestamp":"2026-09-05T10:00:00Z","message_type":"","payload":{"n":$n}}""" + // Cap the channel at 3 so trimming happens on the 4th publish. + val seqs = (1 to 3).map(n => code.api.cache.RedisMessaging.publishMessage(channelName, envelope(n), maxMessages = 3)._1) + seqs should equal(seqs.sorted) + + // A poller that read m1..m3 by offset now asks for offset 3 ... + (4 to 5).foreach(n => code.api.cache.RedisMessaging.publishMessage(channelName, envelope(n), maxMessages = 3)) + val byOffset = reader.fetch(FetchRequest(channelName, 3, 10)) + // ... and gets nothing: the list now holds m3,m4,m5 at positions 0..2. m4 and m5 are lost to it. + byOffset.messages shouldBe empty + byOffset.totalCount should equal(3L) + + // A poller holding m3's sequence gets exactly the two it has not seen. + val byCursor = reader.fetch(FetchRequest(channelName, 0, 10, seqs.last)) + byCursor.messages.map(_.messageId) should equal(Seq("m4", "m5")) + byCursor.hasMore should equal(false) + byCursor.nextAfterSequence should equal(byCursor.latestSequence) + + code.api.cache.RedisMessaging.deleteChannel(channelName) + } + + scenario("Subscribe streams a message published after the stream opened", GrpcSignal) { + if (!redisReachable) cancel("Redis is not reachable from this test JVM") + SignalEventBus.isRunning should equal(true) + val channelName = s"grpc-sub-${java.util.UUID.randomUUID().toString.take(8)}" + + val received = new java.util.concurrent.ConcurrentLinkedQueue[SignalMessage]() + val latch = new CountDownLatch(1) + val observer = new StreamObserver[SignalMessage] { + override def onNext(value: SignalMessage): Unit = { received.add(value); latch.countDown() } + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + } + asyncStub(tokenOf(user1)).subscribe(SubscribeRequest(channelName), observer) + + // The observer is registered on the server inside subscribe(), but the call itself travels + // over the socket: wait until the server-side registration is visible. + val deadline = System.currentTimeMillis() + 10000 + while (SignalEventBus.subscriberCount(channelName) == 0 && System.currentTimeMillis() < deadline) Thread.sleep(50) + SignalEventBus.subscriberCount(channelName) should equal(1) + + val published = blockingStub(tokenOf(user2)).publish(PublishRequest(channelName, "", "hello", """{"hi":"there"}""")) + + latch.await(10, TimeUnit.SECONDS) should equal(true) + val streamed = received.peek() + streamed.messageId should equal(published.messageId) + streamed.channelName should equal(channelName) + streamed.payloadJson should equal("""{"hi":"there"}""") + streamed.senderUserId should equal(resourceUser2.userId) + streamed.sequence should equal(published.sequence) + + code.api.cache.RedisMessaging.deleteChannel(channelName) + } + } +} diff --git a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala index 7b76a27f5e..e544d5af9a 100644 --- a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala +++ b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala @@ -348,6 +348,26 @@ object AccountAccessRequestStatus extends Enumeration { val INITIATED, PENDING, APPROVED, REJECTED = Value } +/** Maker/checker for runtime-supplied code and configuration (dynamic resource docs, connector + * methods, dynamic message docs, ABAC rules ...). A request leaves INITIATED exactly once. FAILED + * means the checker approved but re-validation or apply failed, so the target was not changed. */ +object DynamicChangeRequestStatus extends Enumeration { + type DynamicChangeRequestStatus = Value + val INITIATED, APPROVED, REJECTED, WITHDRAWN, EXPIRED, FAILED = Value +} + +object DynamicChangeRequestOperation extends Enumeration { + type DynamicChangeRequestOperation = Value + val CREATE, UPDATE, DELETE, ACTIVATE, DEACTIVATE = Value +} + +object DynamicChangeRequestTargetType extends Enumeration { + type DynamicChangeRequestTargetType = Value + val DYNAMIC_RESOURCE_DOC, DYNAMIC_MESSAGE_DOC, CONNECTOR_METHOD, ABAC_RULE, + DYNAMIC_ENDPOINT, DYNAMIC_ENTITY, METHOD_ROUTING, ENDPOINT_MAPPING, + WEBUI_PROPS, JSON_SCHEMA_VALIDATION, AUTHENTICATION_TYPE_VALIDATION = Value +} + object AccountRoutingScheme extends Enumeration { type AccountRoutingScheme = Value val IBAN = Value From ff58138d08a9942037ed06ac7def1fef63dfed12 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 6 Sep 2026 09:26:20 +0200 Subject: [PATCH 02/13] apiTagSignalChannel --- obp-api/src/main/scala/code/api/util/ApiTag.scala | 4 +--- .../main/scala/code/api/v6_0_0/APIMethods600.scala | 12 ++++++------ .../src/main/scala/code/api/v6_0_0/Http4s600.scala | 12 ++++++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ApiTag.scala b/obp-api/src/main/scala/code/api/util/ApiTag.scala index 059cbb4aa2..d7d79ef63c 100644 --- a/obp-api/src/main/scala/code/api/util/ApiTag.scala +++ b/obp-api/src/main/scala/code/api/util/ApiTag.scala @@ -171,9 +171,7 @@ object ApiTag { val apiTagBanking = ResourceDocTag("AU-Banking") val apiTagAiAgent = ResourceDocTag("AI-Agent") - val apiTagSignal = ResourceDocTag("Signal") - val apiTagSignalling = ResourceDocTag("Signalling") - val apiTagChannel = ResourceDocTag("Channel") + val apiTagSignalChannel = ResourceDocTag("Signal-Channel") val apiTagFinancialCrime = ResourceDocTag("Financial-Crime") private[this] val tagNameSymbolMapTag: MutableMap[String, ResourceDocTag] = MutableMap() diff --git a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala index 9dab4dfe61..7589327a2f 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala @@ -11073,7 +11073,7 @@ trait APIMethods600 // InvalidSignalChannelName, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel)) +// List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val publishSignalMessage: OBPEndpoint = { // case "signal" :: "channels" :: channelName :: "messages" :: Nil JsonPost json -> _ => @@ -11152,7 +11152,7 @@ trait APIMethods600 // InvalidSignalChannelName, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel)) +// List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalMessages: OBPEndpoint = { // case "signal" :: "channels" :: channelName :: "messages" :: Nil JsonGet _ => @@ -11214,7 +11214,7 @@ trait APIMethods600 // $AuthenticatedUserIsRequired, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel)) +// List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalChannels: OBPEndpoint = { // case "signal" :: "channels" :: Nil JsonGet _ => @@ -11279,7 +11279,7 @@ trait APIMethods600 // InvalidSignalChannelName, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel)) +// List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalChannelInfo: OBPEndpoint = { // case "signal" :: "channels" :: channelName :: "info" :: Nil JsonGet _ => @@ -11330,7 +11330,7 @@ trait APIMethods600 // InvalidSignalChannelName, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel)) +// List(apiTagAiAgent, apiTagSignalChannel)) // // staticResourceDocs += ResourceDoc( // getSignalStats, @@ -11354,7 +11354,7 @@ trait APIMethods600 // UserHasMissingRoles, // UnknownError // ), -// List(apiTagAiAgent, apiTagSignal, apiTagSignalling, apiTagChannel), +// List(apiTagAiAgent, apiTagSignalChannel), // Some(List(canGetSignalStats))) // // lazy val getSignalStats: OBPEndpoint = { diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 4497c5e2ca..ab46cbb1d0 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -10439,7 +10439,7 @@ object Http4s600 { EmptyBody, signalChannelsJsonV600, List($AuthenticatedUserIsRequired, UnknownError), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, None, http4sPartialFunction = Some(getSignalChannels) ) @@ -10460,7 +10460,7 @@ object Http4s600 { EmptyBody, signalChannelInfoJsonV600, List($AuthenticatedUserIsRequired, InvalidSignalChannelName, UnknownError), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, None, http4sPartialFunction = Some(getSignalChannelInfo) ) @@ -10481,7 +10481,7 @@ object Http4s600 { EmptyBody, signalStatsJsonV600, List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, Some(canGetSignalStats :: Nil), http4sPartialFunction = Some(getSignalStats) ) @@ -10532,7 +10532,7 @@ object Http4s600 { SignalMessageContainsDangerousCharacters, UnknownError ), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, None, http4sPartialFunction = Some(publishSignalMessage) ) @@ -10569,7 +10569,7 @@ object Http4s600 { EmptyBody, signalMessagesJsonV600, List($AuthenticatedUserIsRequired, InvalidSignalChannelName, InvalidNumber, UnknownError), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, None, http4sPartialFunction = Some(getSignalMessages) ) @@ -10595,7 +10595,7 @@ object Http4s600 { // the CanDeleteSignalChannel role gate was added after the migration — // an ungated delete let any authenticated user destroy any channel. List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidSignalChannelName, UnknownError), - apiTagAiAgent :: apiTagSignal :: apiTagSignalling :: apiTagChannel :: Nil, + apiTagAiAgent :: apiTagSignalChannel :: Nil, Some(canDeleteSignalChannel :: Nil), http4sPartialFunction = Some(deleteSignalChannel) ) From c9bc5c3e4814affba41bc83875132ec077cfc2eb Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 6 Sep 2026 09:56:24 +0200 Subject: [PATCH 03/13] signal-channels path --- obp-api/src/main/protobuf/signal.proto | 6 ++-- .../resources/props/sample.props.template | 2 +- .../main/scala/code/api/util/Glossary.scala | 2 +- .../scala/code/api/v6_0_0/APIMethods600.scala | 24 ++++++------- .../scala/code/api/v6_0_0/Http4s600.scala | 36 +++++++++---------- .../api/SignalChannelsServiceGrpc.scala | 6 ++-- .../code/signal/SignalContentPolicy.scala | 2 +- .../code/api/sweep/SuccessSweepTest.scala | 2 +- .../code/api/v6_0_0/SignalChannelTest.scala | 20 +++++------ 9 files changed, 50 insertions(+), 50 deletions(-) diff --git a/obp-api/src/main/protobuf/signal.proto b/obp-api/src/main/protobuf/signal.proto index a8bf0a8b2b..8c854a4e0d 100644 --- a/obp-api/src/main/protobuf/signal.proto +++ b/obp-api/src/main/protobuf/signal.proto @@ -25,7 +25,7 @@ message SignalChannelInfo { int64 ttl_seconds = 3; } -// --- Publish: 1:1 with POST /signal/channels/{name}/messages --- +// --- Publish: 1:1 with POST /signal-channels/{name}/messages --- message PublishRequest { string channel_name = 1; @@ -42,7 +42,7 @@ message PublishResponse { int64 sequence = 5; } -// --- Fetch: 1:1 with GET /signal/channels/{name}/messages --- +// --- Fetch: 1:1 with GET /signal-channels/{name}/messages --- // Privacy filter applied server-side: caller sees broadcasts plus messages // to/from themselves. Same logic as REST. @@ -64,7 +64,7 @@ message FetchResponse { int64 next_after_sequence = 6; // pass back as after_sequence to continue (advances past hidden private messages too) } -// --- ListChannels: 1:1 with GET /signal/channels --- +// --- ListChannels: 1:1 with GET /signal-channels --- // Returns broadcast-visible channels only, matching REST behaviour. message ListChannelsRequest {} diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 277b40800c..312f9390bc 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1359,7 +1359,7 @@ database_messages_scheduler_interval=3600 # chat.email_digest_active_grace_minutes = 10 # Signal channels ----------------------------------------------------------- -# Redis-backed ephemeral channels (/signal/channels endpoints) for lightweight +# Redis-backed ephemeral channels (/signal-channels endpoints) for lightweight # agent-to-agent coordination. Per-channel TTL (refreshed on every publish) # and per-channel message cap: # messaging.channel.ttl.seconds = 3600 diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index df4f28b492..e825e5525d 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -6067,7 +6067,7 @@ object Glossary extends MdcLoggable { |Signal channels are readable and writable by any authenticated consumer on the instance. If your agent feeds received payloads to an LLM, treat them as **untrusted data, never as instructions** — the character checks above stop display-layer trickery, but no server-side check can stop a payload from *saying* something misleading. Prompt-injection defence belongs in the consuming agent. | |## Endpoints - |See the API Explorer tags **Signal** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal/channels/...`. + |See the API Explorer tags **Signal-Channel** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal-channels/...`. | |## gRPC |The same operations are served over gRPC by `SignalChannelsService` (package `code.obp.grpc.signal.g1`, contract in `signal.proto`) when the gRPC server is enabled (`grpc.server.enabled`): **Publish**, **Fetch** and **ListChannels** are 1:1 with the REST endpoints and share their storage, and **Subscribe** is a server-side stream of new messages on one channel. Subscribe is live only — no catch-up, no replay — and applies the same privacy filter as Fetch. Each publish, REST or gRPC, is pushed to subscribers through Redis pub/sub. Authenticate with the same `Authorization` value the REST endpoints take, sent as gRPC metadata. diff --git a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala index 7589327a2f..fe9f5b6374 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala @@ -11047,7 +11047,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(publishSignalMessage), // "POST", -// "/signal/channels/CHANNEL_NAME/messages", +// "/signal-channels/CHANNEL_NAME/messages", // "Publish Signal Message", // s"""Publish a message to a signal channel. // | @@ -11076,7 +11076,7 @@ trait APIMethods600 // List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val publishSignalMessage: OBPEndpoint = { -// case "signal" :: "channels" :: channelName :: "messages" :: Nil JsonPost json -> _ => +// case "signal-channels" :: channelName :: "messages" :: Nil JsonPost json -> _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { @@ -11128,7 +11128,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(getSignalMessages), // "GET", -// "/signal/channels/CHANNEL_NAME/messages", +// "/signal-channels/CHANNEL_NAME/messages", // "Get Signal Messages", // s"""Fetch messages from a signal channel with offset/limit pagination. // | @@ -11155,7 +11155,7 @@ trait APIMethods600 // List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalMessages: OBPEndpoint = { -// case "signal" :: "channels" :: channelName :: "messages" :: Nil JsonGet _ => +// case "signal-channels" :: channelName :: "messages" :: Nil JsonGet _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { @@ -11196,7 +11196,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(getSignalChannels), // "GET", -// "/signal/channels", +// "/signal-channels", // "List Signal Channels", // s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. // |Messages are ephemeral and will expire after the configured TTL (default 1 hour). @@ -11217,7 +11217,7 @@ trait APIMethods600 // List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalChannels: OBPEndpoint = { -// case "signal" :: "channels" :: Nil JsonGet _ => +// case "signal-channels" :: Nil JsonGet _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { @@ -11262,7 +11262,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(getSignalChannelInfo), // "GET", -// "/signal/channels/CHANNEL_NAME/info", +// "/signal-channels/CHANNEL_NAME/info", // "Get Signal Channel Info", // s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. // |Messages are ephemeral and will expire after the configured TTL (default 1 hour). @@ -11282,7 +11282,7 @@ trait APIMethods600 // List(apiTagAiAgent, apiTagSignalChannel)) // // lazy val getSignalChannelInfo: OBPEndpoint = { -// case "signal" :: "channels" :: channelName :: "info" :: Nil JsonGet _ => +// case "signal-channels" :: channelName :: "info" :: Nil JsonGet _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { @@ -11313,7 +11313,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(deleteSignalChannel), // "DELETE", -// "/signal/channels/CHANNEL_NAME", +// "/signal-channels/CHANNEL_NAME", // "Delete Signal Channel", // s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. // |Messages are ephemeral and will expire after the configured TTL (default 1 hour). @@ -11337,7 +11337,7 @@ trait APIMethods600 // implementedInApiVersion, // nameOf(getSignalStats), // "GET", -// "/signal/channels/stats", +// "/signal-channels/stats", // "Get Signal Channel Stats", // s"""Returns statistics for all signal channels, including private-only channels. // | @@ -11358,7 +11358,7 @@ trait APIMethods600 // Some(List(canGetSignalStats))) // // lazy val getSignalStats: OBPEndpoint = { -// case "signal" :: "channels" :: "stats" :: Nil JsonGet _ => +// case "signal-channels" :: "stats" :: Nil JsonGet _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { @@ -11393,7 +11393,7 @@ trait APIMethods600 // // // lazy val deleteSignalChannel: OBPEndpoint = { -// case "signal" :: "channels" :: channelName :: Nil JsonDelete _ => +// case "signal-channels" :: channelName :: Nil JsonDelete _ => // cc => // implicit val ec = EndpointContext(Some(cc)) // for { diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index ab46cbb1d0..f1c59ad525 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -2857,18 +2857,18 @@ object Http4s600 { // ─── Phase 2: Signal bucket (6 endpoints) ──────────────────────────── - // GET /obp/v6.0.0/signal/channels + // GET /obp/v6.0.0/signal-channels lazy val getSignalChannels: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ GET -> `prefixPath` / "signal" / "channels" => + case req @ GET -> `prefixPath` / "signal-channels" => EndpointHelpers.withUser(req) { (_, cc) => // Shared with the gRPC SignalChannelsService.ListChannels, see code.signal.SignalChannels Future(SignalChannelsJsonV600(code.signal.SignalChannels.listBroadcastChannels())) } } - // GET /obp/v6.0.0/signal/channels/CHANNEL_NAME/info + // GET /obp/v6.0.0/signal-channels/CHANNEL_NAME/info lazy val getSignalChannelInfo: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ GET -> `prefixPath` / "signal" / "channels" / channelName / "info" => + case req @ GET -> `prefixPath` / "signal-channels" / channelName / "info" => EndpointHelpers.withUser(req) { (_, cc) => for { _ <- Helper.booleanToFuture(InvalidSignalChannelName, cc = Some(cc)) { @@ -2889,9 +2889,9 @@ object Http4s600 { } } - // GET /obp/v6.0.0/signal/channels/stats + // GET /obp/v6.0.0/signal-channels/stats lazy val getSignalStats: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ GET -> `prefixPath` / "signal" / "channels" / "stats" => + case req @ GET -> `prefixPath` / "signal-channels" / "stats" => EndpointHelpers.withUser(req) { (_, cc) => Future { val names = code.api.cache.RedisMessaging.listChannels() @@ -2908,9 +2908,9 @@ object Http4s600 { } } - // POST /obp/v6.0.0/signal/channels/CHANNEL_NAME/messages (201) + // POST /obp/v6.0.0/signal-channels/CHANNEL_NAME/messages (201) lazy val publishSignalMessage: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ POST -> `prefixPath` / "signal" / "channels" / channelName / "messages" => + case req @ POST -> `prefixPath` / "signal-channels" / channelName / "messages" => EndpointHelpers.executeFutureCreated(req) { implicit val cc: CallContext = req.callContext val rawBody = cc.httpBody.getOrElse("") @@ -2942,9 +2942,9 @@ object Http4s600 { } } - // GET /obp/v6.0.0/signal/channels/CHANNEL_NAME/messages + // GET /obp/v6.0.0/signal-channels/CHANNEL_NAME/messages lazy val getSignalMessages: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ GET -> `prefixPath` / "signal" / "channels" / channelName / "messages" => + case req @ GET -> `prefixPath` / "signal-channels" / channelName / "messages" => EndpointHelpers.withUser(req) { (user, cc) => for { _ <- Helper.booleanToFuture(InvalidSignalChannelName, cc = Some(cc)) { @@ -2969,9 +2969,9 @@ object Http4s600 { } } - // DELETE /obp/v6.0.0/signal/channels/CHANNEL_NAME (200 with body — not 204) + // DELETE /obp/v6.0.0/signal-channels/CHANNEL_NAME (200 with body — not 204) lazy val deleteSignalChannel: HttpRoutes[IO] = HttpRoutes.of[IO] { - case req @ DELETE -> `prefixPath` / "signal" / "channels" / channelName => + case req @ DELETE -> `prefixPath` / "signal-channels" / channelName => EndpointHelpers.executeAndRespond(req) { implicit cc => for { _ <- Helper.booleanToFuture(InvalidSignalChannelName, cc = Some(cc)) { @@ -10424,7 +10424,7 @@ object Http4s600 { implementedInApiVersion, nameOf(getSignalChannels), "GET", - "/signal/channels", + "/signal-channels", "List Signal Channels", s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. |Messages are ephemeral and will expire after the configured TTL (default 1 hour). @@ -10447,7 +10447,7 @@ object Http4s600 { implementedInApiVersion, nameOf(getSignalChannelInfo), "GET", - "/signal/channels/CHANNEL_NAME/info", + "/signal-channels/CHANNEL_NAME/info", "Get Signal Channel Info", s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. |Messages are ephemeral and will expire after the configured TTL (default 1 hour). @@ -10468,7 +10468,7 @@ object Http4s600 { implementedInApiVersion, nameOf(getSignalStats), "GET", - "/signal/channels/stats", + "/signal-channels/stats", "Get Signal Channel Stats", s"""Returns statistics for all signal channels, including private-only channels. | @@ -10489,7 +10489,7 @@ object Http4s600 { implementedInApiVersion, nameOf(publishSignalMessage), "POST", - "/signal/channels/CHANNEL_NAME/messages", + "/signal-channels/CHANNEL_NAME/messages", "Publish Signal Message", s"""Publish a message to a signal channel. | @@ -10540,7 +10540,7 @@ object Http4s600 { implementedInApiVersion, nameOf(getSignalMessages), "GET", - "/signal/channels/CHANNEL_NAME/messages", + "/signal-channels/CHANNEL_NAME/messages", "Get Signal Messages", s"""Fetch messages from a signal channel with offset/limit pagination. | @@ -10577,7 +10577,7 @@ object Http4s600 { implementedInApiVersion, nameOf(deleteSignalChannel), "DELETE", - "/signal/channels/CHANNEL_NAME", + "/signal-channels/CHANNEL_NAME", "Delete Signal Channel", s"""Signal channels provide short-lived, Redis-backed messaging designed for AI agent discovery and coordination, but usable by any authenticated OBP consumer. |Messages are ephemeral and will expire after the configured TTL (default 1 hour). diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala index f7307c3660..3b130aefc9 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalChannelsServiceGrpc.scala @@ -59,13 +59,13 @@ object SignalChannelsServiceGrpc { trait SignalChannelsService extends _root_.scalapb.grpc.AbstractService { override def serviceCompanion = SignalChannelsService - /** 1:1 with POST /signal/channels/CHANNEL_NAME/messages */ + /** 1:1 with POST /signal-channels/CHANNEL_NAME/messages */ def publish(request: PublishRequest): scala.concurrent.Future[PublishResponse] - /** 1:1 with GET /signal/channels/CHANNEL_NAME/messages (offset/limit, privacy-filtered) */ + /** 1:1 with GET /signal-channels/CHANNEL_NAME/messages (offset/limit, privacy-filtered) */ def fetch(request: FetchRequest): scala.concurrent.Future[FetchResponse] - /** 1:1 with GET /signal/channels (broadcast-visible channels only) */ + /** 1:1 with GET /signal-channels (broadcast-visible channels only) */ def listChannels(request: ListChannelsRequest): scala.concurrent.Future[ListChannelsResponse] /** Server-side stream of new messages on one channel. Live only: no catch-up, no replay. */ diff --git a/obp-api/src/main/scala/code/signal/SignalContentPolicy.scala b/obp-api/src/main/scala/code/signal/SignalContentPolicy.scala index 1ff67dc236..77316899ab 100644 --- a/obp-api/src/main/scala/code/signal/SignalContentPolicy.scala +++ b/obp-api/src/main/scala/code/signal/SignalContentPolicy.scala @@ -6,7 +6,7 @@ import org.json4s.JsonAST._ /** * Content policy for signal channel messages (Redis-backed agent-to-agent - * coordination — see RedisMessaging and the /signal/channels endpoints). + * coordination — see RedisMessaging and the /signal-channels endpoints). * * Signal payloads are machine-consumed data, so the policy differs from chat * on purpose: nothing is ever rewritten (agents may hash, sign, or diff --git a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala index 66502a2349..05202186fb 100644 --- a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala @@ -116,7 +116,7 @@ class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers with Sw * EndpointCatalog.hasPlaceholder, not a local copy of the rule. This used to hold its own, * and the two had already drifted: the catalog substitutes any segment ending in ID, _CODE or * _NAME, this one looked for _ID and _CODE and had never learnt about _NAME. So - * `/signal/channels/CHANNEL_NAME/info` was a placeholder to the catalog -- which duly replaced + * `/signal-channels/CHANNEL_NAME/info` was a placeholder to the catalog -- which duly replaced * it with a channel that does not exist -- and NOT a placeholder here, so this suite selected * it as an endpoint that "needs nothing created first" and then failed it for answering 404. * diff --git a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala index a3aa26ba2c..426ca37d9d 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala @@ -11,7 +11,7 @@ import org.json4s.JsonAST.JValue import org.scalatest.Tag /** - * Validation / error-path tests for the /signal/channels endpoints. + * Validation / error-path tests for the /signal-channels endpoints. * * Every scenario here fails BEFORE RedisMessaging is touched (auth and role * checks run in the middleware; the size and character checks run in the @@ -30,10 +30,10 @@ class SignalChannelTest extends V600ServerSetup { try jedis.ping() finally jedis.close() }.isSuccess - private def publishRequest = (v6_0_0_Request / "signal" / "channels" / "test-channel" / "messages").POST - private def deleteRequest = (v6_0_0_Request / "signal" / "channels" / "test-channel").DELETE + private def publishRequest = (v6_0_0_Request / "signal-channels" / "test-channel" / "messages").POST + private def deleteRequest = (v6_0_0_Request / "signal-channels" / "test-channel").DELETE - feature(s"Publish Signal Message - POST /obp/v6.0.0/signal/channels/CHANNEL_NAME/messages - $VersionOfApi") { + feature(s"Publish Signal Message - POST /obp/v6.0.0/signal-channels/CHANNEL_NAME/messages - $VersionOfApi") { scenario("Anonymous access should fail with 401", ApiEndpointPublish, VersionOfApi) { val response = makePostRequest(publishRequest, """{"payload":{"hello":"world"}}""") @@ -76,7 +76,7 @@ class SignalChannelTest extends V600ServerSetup { // AFTER the size and character checks without reaching Redis — proving // legitimate international text is not rejected as dangerous. val longName = "a" * 129 - val request = (v6_0_0_Request / "signal" / "channels" / longName / "messages").POST <@ (user1) + val request = (v6_0_0_Request / "signal-channels" / longName / "messages").POST <@ (user1) val body = """{"payload":{"note":"Grüße aus Berlin, 東京"}}""" val response = makePostRequest(request, body) response.code should equal(400) @@ -84,10 +84,10 @@ class SignalChannelTest extends V600ServerSetup { } } - feature(s"Get Signal Messages - GET /obp/v6.0.0/signal/channels/CHANNEL_NAME/messages - $VersionOfApi") { + feature(s"Get Signal Messages - GET /obp/v6.0.0/signal-channels/CHANNEL_NAME/messages - $VersionOfApi") { scenario("a non-numeric after_sequence should fail with 400 OBP-10002", ApiEndpointGetMessages, VersionOfApi) { - val request = (v6_0_0_Request / "signal" / "channels" / "test-channel" / "messages").GET <@ (user1) < Date: Sun, 6 Sep 2026 11:29:14 +0200 Subject: [PATCH 04/13] Fix test related to dynamicCodeOn --- .../scala/code/api/v7_0_0/DynamicChangeRequestTest.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala index 058b6ae3fe..36f9e48fe9 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala @@ -46,7 +46,10 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { def dynamicEndpoint = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString // The local test.default.props may not enable dynamic code (CI does); these scenarios need it on. - private def dynamicCodeOn(): Unit = setPropsValues("allow_user_generated_scala_code" -> "true") + // Braced body on purpose: .github/scripts/check_test_isolation.py only recognises `def name {` as a helper. + private def dynamicCodeOn(): Unit = { + setPropsValues("allow_user_generated_scala_code" -> "true") + } private def enableMakerChecker(): Unit = { dynamicCodeOn() From 5c29a1d423d02fb02d20b2bd3c8983b581b1da9e Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 6 Sep 2026 21:44:06 +0200 Subject: [PATCH 05/13] AuthRateLimiter and SelfServiceRateLimiter and related error messages. Also glossary item to explain the three rate limiters. --- .../resources/props/sample.props.template | 47 ++++ .../main/scala/code/api/GatewayLogin.scala | 8 +- obp-api/src/main/scala/code/api/dauth.scala | 8 +- .../src/main/scala/code/api/directlogin.scala | 2 +- obp-api/src/main/scala/code/api/siwe.scala | 2 +- .../main/scala/code/api/util/APIUtil.scala | 3 +- .../scala/code/api/util/AuthRateLimiter.scala | 34 ++- .../scala/code/api/util/ErrorMessages.scala | 12 + .../main/scala/code/api/util/Glossary.scala | 131 ++++++++++- .../api/util/SelfServiceRateLimiter.scala | 212 ++++++++++++++++++ .../code/api/util/http4s/Http4sApp.scala | 6 +- .../code/api/util/http4s/Http4sSupport.scala | 4 + .../SelfServiceRateLimitMiddleware.scala | 121 ++++++++++ .../scala/code/api/v5_1_0/Http4s510.scala | 5 + .../scala/code/api/v6_0_0/Http4s600.scala | 22 ++ .../scala/code/api/v7_0_0/Http4s700.scala | 125 ++++++++++- .../code/api/v7_0_0/JSONFactory7.0.0.scala | 172 +++++++++++++- .../signal/SignalChannelsServiceImpl.scala | 14 ++ .../api/util/SelfServiceRateLimiterTest.scala | 191 ++++++++++++++++ .../code/api/v7_0_0/RateLimitersTest.scala | 66 ++++++ 20 files changed, 1157 insertions(+), 28 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala create mode 100644 obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala create mode 100644 obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index c74f6eb632..0538a23846 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1171,6 +1171,53 @@ featured_apis=elasticSearchWarehouseV300 # rate_limiting_per_day = -1 # rate_limiting_per_week = -1 # rate_limiting_per_month = -1 +# +# -- Three rate limiters -- +# OBP has three rate limiters. Each answers 429 with its own error code so a client knows which one it hit: +# 1. Self-service (self_service.rate_limit.*) runs first, before routing and authentication, keyed by +# client IP, on the endpoints anyone can call before the bank has granted them anything -> OBP-10060 +# 2. Authentication (auth.rate_limit.*) runs inside the credential check of DirectLogin, DAuth, +# GatewayLogin and SIWE, keyed by IP and by account, against brute force and lockout -> OBP-10061 +# 3. Consumer quota (rate_limiting_per_* and the rate limit rows written by the management endpoints and +# API Product Subscriptions) runs after authentication, keyed by consumer_id -> OBP-10018 +# +# -- Authentication rate limiting (per IP and per account, before the password is checked) -- +# Off by default. When enabled, shadow mode logs trips (event=auth_rate_limit_shadow_trip) and allows the +# attempt; enforce mode answers 429 OBP-10061. Counters live in Redis and fail open. +# auth.rate_limit.enabled = false +# auth.rate_limit.mode = shadow +# auth.rate_limit.per_ip.per_minute = 10 +# auth.rate_limit.per_ip.per_hour = 100 +# auth.rate_limit.per_user.per_minute = 6 +# +# -- Self-service rate limiting (per client IP, before any credential) -- +# Applies to the endpoints a caller can use before it holds credentials, grouped in scopes. +# Logins are not a scope here; the authentication limiter above counts them. +# signup POST /users, /users/email-validation, /banks/BANK_ID/user-invitations +# password_reset POST /users/password-reset-url, /users/password +# consent_request POST /consumer/consent-requests, /consumer/vrp-consent-requests +# consumer_registration POST /dynamic-registration/consumers +# lookup POST /account/check/scheme/iban +# signal_channel_create POST /signal-channels/CHANNEL_NAME/messages when the channel does not exist yet +# Enabled by default in shadow mode: a trip is logged (event=self_service_rate_limit_shadow_trip) +# and reported to the caller in the X-Rate-Limit-Warning header (OBP-10059), but the request is +# allowed. Set mode to enforce to answer 429 OBP-10060 instead. Counters live in Redis and fail open. +# self_service.rate_limit.enabled = true +# self_service.rate_limit.mode = shadow +# Generic per-IP limits; -1 switches a window off, 0 blocks every call in it. +# Built-in per-scope defaults (applied when neither a scope prop nor a generic prop is set): +# signup 3/5/10 | password_reset 3/5/10 | consent_request 10/30/100 +# consumer_registration 5/10/20 | lookup 20/60/200 | signal_channel_create 5/20/50 +# self_service.rate_limit.per_ip.per_minute = 10 +# self_service.rate_limit.per_ip.per_hour = 60 +# self_service.rate_limit.per_ip.per_day = 200 +# Per-scope overrides, e.g.: +# self_service.rate_limit.signup.per_ip.per_hour = 5 +# Global per-hour cap per scope across all IPs (circuit breaker against a distributed spray). +# Built-in defaults: 500 for signup, password_reset and consumer_registration; off (-1) elsewhere. +# self_service.rate_limit.signup.global.per_hour = 500 +# Optional text appended to the warning when you have announced an enforcement date. +# self_service.rate_limit.enforce_announced_from = 2026-10-01 # ----------------------------------------------------- # -- Migration Scripts ---------------------------- diff --git a/obp-api/src/main/scala/code/api/GatewayLogin.scala b/obp-api/src/main/scala/code/api/GatewayLogin.scala index 94dbcd4f7c..c769d66e76 100755 --- a/obp-api/src/main/scala/code/api/GatewayLogin.scala +++ b/obp-api/src/main/scala/code/api/GatewayLogin.scala @@ -244,8 +244,8 @@ object GatewayLogin extends MdcLoggable { logger.debug("login_user_name: " + username) // Pre-credential rate limit. Disabled by default; controlled via auth.rate_limit.* props. // In shadow mode trips are logged and Right is returned; only enforce mode produces Left. - AuthRateLimiter.check(APIUtil.getRemoteIpAddress(), gateway, username) match { - case Left(_) => return Failure(ErrorMessages.TooManyRequests) + AuthRateLimiter.check(callContext.map(_.ipAddress).filter(_.nonEmpty).getOrElse(APIUtil.getRemoteIpAddress()), gateway, username) match { + case Left(_) => return Failure(ErrorMessages.TooManyRequestsAuth) case Right(_) => // continue } val cbsAndCallContextBox = refreshBankAccounts(jwtPayload, callContext) @@ -314,8 +314,8 @@ object GatewayLogin extends MdcLoggable { val consentId = if (jti.isEmpty) None else Some(jti) logger.debug("login_user_name: " + username) // Pre-credential rate limit. Disabled by default; controlled via auth.rate_limit.* props. - AuthRateLimiter.check(APIUtil.getRemoteIpAddress(), gateway, username) match { - case Left(_) => return Future.successful(Failure(ErrorMessages.TooManyRequests)) + AuthRateLimiter.check(callContext.map(_.ipAddress).filter(_.nonEmpty).getOrElse(APIUtil.getRemoteIpAddress()), gateway, username) match { + case Left(_) => return Future.successful(Failure(ErrorMessages.TooManyRequestsAuth)) case Right(_) => // continue } val cbsAndCallContextF = refreshBankAccountsFuture(jwtPayload, callContext) diff --git a/obp-api/src/main/scala/code/api/dauth.scala b/obp-api/src/main/scala/code/api/dauth.scala index 3dccf258ba..f432319c12 100755 --- a/obp-api/src/main/scala/code/api/dauth.scala +++ b/obp-api/src/main/scala/code/api/dauth.scala @@ -142,8 +142,8 @@ object DAuth extends MdcLoggable { val provider = "dauth."+getFieldFromPayloadJson(jwtPayload, "network_name") logger.debug("login_user_name: " + userName) // Pre-credential rate limit. Disabled by default; controlled via auth.rate_limit.* props. - AuthRateLimiter.check(APIUtil.getRemoteIpAddress(), provider, userName) match { - case Left(_) => return Failure(ErrorMessages.TooManyRequests) + AuthRateLimiter.check(callContext.map(_.ipAddress).filter(_.nonEmpty).getOrElse(APIUtil.getRemoteIpAddress()), provider, userName) match { + case Left(_) => return Failure(ErrorMessages.TooManyRequestsAuth) case Right(_) => // continue } for { @@ -167,8 +167,8 @@ object DAuth extends MdcLoggable { val provider = "dauth."+ getFieldFromPayloadJson(jwtPayload, "network_name") logger.debug("login_user_name: " + username) // Pre-credential rate limit. Disabled by default; controlled via auth.rate_limit.* props. - AuthRateLimiter.check(APIUtil.getRemoteIpAddress(), provider, username) match { - case Left(_) => return Future.successful(Failure(ErrorMessages.TooManyRequests)) + AuthRateLimiter.check(callContext.map(_.ipAddress).filter(_.nonEmpty).getOrElse(APIUtil.getRemoteIpAddress()), provider, username) match { + case Left(_) => return Future.successful(Failure(ErrorMessages.TooManyRequestsAuth)) case Right(_) => // continue } diff --git a/obp-api/src/main/scala/code/api/directlogin.scala b/obp-api/src/main/scala/code/api/directlogin.scala index 01e9b66c4e..9bc3ab4ee6 100644 --- a/obp-api/src/main/scala/code/api/directlogin.scala +++ b/obp-api/src/main/scala/code/api/directlogin.scala @@ -134,7 +134,7 @@ object DirectLogin extends MdcLoggable { message = ErrorMessages.UserEmailNotValidated httpCode = 401 } else if (userId == AuthUser.rateLimitExceededStateCode) { - message = ErrorMessages.TooManyRequests + message = ErrorMessages.TooManyRequestsAuth httpCode = 429 } else { val jwtPayloadAsJson = diff --git a/obp-api/src/main/scala/code/api/siwe.scala b/obp-api/src/main/scala/code/api/siwe.scala index c5d29fa202..95e980dca1 100644 --- a/obp-api/src/main/scala/code/api/siwe.scala +++ b/obp-api/src/main/scala/code/api/siwe.scala @@ -257,7 +257,7 @@ object SIWE extends MdcLoggable { def getOrCreateUser(chainId: Long, checksumAddress: String): Box[User] = { val provider = "siwe." + chainId AuthRateLimiter.check(APIUtil.getRemoteIpAddress(), provider, checksumAddress) match { - case Left(_) => Failure(ErrorMessages.TooManyRequests) + case Left(_) => Failure(ErrorMessages.TooManyRequestsAuth) case Right(_) => UserX.getOrCreateDauthResourceUser(provider, checksumAddress) } } diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 90dbe59909..adec0652fd 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -781,7 +781,8 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ message.contains(extractErrorMessageCode(requestTimeout)) } def check429(message: String): Boolean = { - message.contains(extractErrorMessageCode(TooManyRequests)) + List(TooManyRequests, TooManyRequestsSelfService, TooManyRequestsAuth) + .exists(m => message.contains(extractErrorMessageCode(m))) } val (code, responseHeaders) = message match { diff --git a/obp-api/src/main/scala/code/api/util/AuthRateLimiter.scala b/obp-api/src/main/scala/code/api/util/AuthRateLimiter.scala index ba31327cda..efc0d14ab4 100644 --- a/obp-api/src/main/scala/code/api/util/AuthRateLimiter.scala +++ b/obp-api/src/main/scala/code/api/util/AuthRateLimiter.scala @@ -7,9 +7,14 @@ import code.util.Helper.MdcLoggable /** Pre-credential-check rate limiter for authentication endpoints. * - * Distinct from [[RateLimitingUtil]] (which fires post-auth, keyed by consumer_id). + * One of three limiters, each with its own 429 code: + * - [[RateLimitingUtil]] post-auth, keyed by Consumer, the commercial quota -> OBP-10018 + * - [[SelfServiceRateLimiter]] pre-auth, keyed by client IP, abuse of the free tier -> OBP-10060 + * - this limiter inside the credential check, keyed by IP and account -> OBP-10061 * This limiter fires BEFORE the password check, keyed by source IP and (provider, username), - * to defend against brute-force, credential-stuffing, and lockout-DoS attacks. + * to defend against brute-force, credential-stuffing, and lockout-DoS attacks. It owns every + * credential check: DirectLogin (via AuthUser.getResourceUserId), DAuth, GatewayLogin and SIWE. + * Callers render a trip as 429 with ErrorMessages.TooManyRequestsAuth. * * Three counters checked per call: * - per-IP per minute — burst defence @@ -30,28 +35,35 @@ object AuthRateLimiter extends MdcLoggable { /** A counter that exceeded its limit. Carries the data the caller needs to render a 429. */ case class Exceeded(counter: String, current: Long, limit: Long, retryAfterSeconds: Long) - private def enabled: Boolean = APIUtil.getPropsAsBoolValue("auth.rate_limit.enabled", false) - private def mode: String = APIUtil.getPropsValue("auth.rate_limit.mode", "shadow") - private def perIpPerMinute: Long = APIUtil.getPropsAsLongValue("auth.rate_limit.per_ip.per_minute", 10L) - private def perIpPerHour: Long = APIUtil.getPropsAsLongValue("auth.rate_limit.per_ip.per_hour", 100L) - private def perUserPerMinute: Long = APIUtil.getPropsAsLongValue("auth.rate_limit.per_user.per_minute", 6L) + // Public so GET /obp/v7.0.0/management/rate-limiter-config can report the live configuration. + val PropsPrefix = "auth.rate_limit" + def enabled: Boolean = APIUtil.getPropsAsBoolValue(s"$PropsPrefix.enabled", false) + def mode: String = APIUtil.getPropsValue(s"$PropsPrefix.mode", "shadow") + def perIpPerMinute: Long = APIUtil.getPropsAsLongValue(s"$PropsPrefix.per_ip.per_minute", 10L) + def perIpPerHour: Long = APIUtil.getPropsAsLongValue(s"$PropsPrefix.per_ip.per_hour", 100L) + def perUserPerMinute: Long = APIUtil.getPropsAsLongValue(s"$PropsPrefix.per_user.per_minute", 6L) def check(ip: String, provider: String, username: String): Either[Exceeded, Unit] = { if (!enabled) return Right(()) - val safeIp = if (ip == null || ip.isEmpty) "unknown" else ip + // "Unknown" is what APIUtil.getRemoteIpAddress() returns when no request is in scope. Pooling + // every such call under one "ip_Unknown" key would rate limit all users together, so an + // unknown IP disables the per-IP windows; the per-user window still applies. + val ipKnown = ip != null && ip.trim.nonEmpty && !ip.equalsIgnoreCase("unknown") + val safeIp = if (ipKnown) ip.trim else "unknown" val safeProvider = if (provider == null || provider.isEmpty) "local" else provider val safeUser = if (username == null) "" else username val userHash = sha256Hex(safeUser).take(12) val counters: List[(String, String, LimitCallPeriod, Long)] = List( - ("ip_per_minute", buildKey(s"ip_$safeIp", PER_MINUTE), PER_MINUTE, perIpPerMinute), + ("ip_per_minute", buildKey(s"ip_$safeIp", PER_MINUTE), PER_MINUTE, if (ipKnown) perIpPerMinute else -1L), ("user_per_minute", buildKey(s"user_${safeProvider}_$userHash", PER_MINUTE), PER_MINUTE, perUserPerMinute), - ("ip_per_hour", buildKey(s"ip_$safeIp", PER_HOUR), PER_HOUR, perIpPerHour) + ("ip_per_hour", buildKey(s"ip_$safeIp", PER_HOUR), PER_HOUR, if (ipKnown) perIpPerHour else -1L) ) val trips = counters.flatMap { case (name, key, period, limit) => - val (ttl, current) = RateLimitingUtil.incrementCounter(key, period) + // limit < 0: window switched off (unknown IP), do not touch Redis + val (ttl, current) = if (limit < 0) (-1L, -1L) else RateLimitingUtil.incrementCounter(key, period) // current == -1 signals Redis-unavailable; fail open by skipping this counter. // limit <= 0 signals "disabled"; skip. if (current > 0 && limit > 0 && current > limit) { diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 44c489c6ab..e4e2829912 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -103,6 +103,16 @@ object ErrorMessages { val InvalidFilterParameterFormat = "OBP-10016: Incorrect filter Parameters in URL. " val InvalidUrl = "OBP-10017: Incorrect URL Format. " val TooManyRequests = "OBP-10018: Too Many Requests." + // Three rate limiters, three codes, so a client can tell which counter it hit: + // OBP-10018 the Consumer quota (RateLimitingUtil, post-auth, keyed by consumer_id or anonymous IP) + // OBP-10060 the self-service limiter (SelfServiceRateLimiter, pre-auth, keyed by client IP) + // OBP-10061 the authentication limiter (AuthRateLimiter, inside the credential check, keyed by IP and account) + val TooManyRequestsSelfService = "OBP-10060: Too Many Requests for a self-service endpoint." + val TooManyRequestsAuth = "OBP-10061: Too Many Requests for authentication. Too many login attempts from this address or for this account." + // Not an error: the text of the X-Rate-Limit-Warning header a self-service endpoint returns in + // shadow mode. SCOPE and LIMIT are replaced at runtime, e.g. "signup" and "5 per hour". + // See SelfServiceRateLimiter.warningMessage. + val RateLimitFutureWarning = "OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate limit for SCOPE (LIMIT) in the future." val InvalidBoolean = "OBP-10019: Invalid Boolean. Could not convert value to a boolean type." val InvalidJsonContent = "OBP-10020: Incorrect json." val InvalidConnectorName = "OBP-10021: Incorrect Connector name." @@ -1089,6 +1099,8 @@ object ErrorMessages { DynamicEndpointNotFoundByDynamicEndpointId -> 404, // NotImplemented -> 501, // 400 or 501 TooManyRequests -> 429, + TooManyRequestsSelfService -> 429, + TooManyRequestsAuth -> 429, ResourceDoesNotExist -> 404, AuthenticatedUserIsRequired -> 401, DirectLoginInvalidToken -> 401, diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 4f04e60d72..fcf90bfe99 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -495,6 +495,39 @@ object Glossary extends MdcLoggable { | |This glossary item is Work In Progress. | + | + |### Three rate limiters + | + |OBP runs three independent rate limiters. They are checked in this order, and each answers **429** with its own error code so a client can tell which counter it hit: + | + |1. **Self-service limiter** (`self_service.rate_limit.*`) runs first, before routing and before any authentication, keyed by the client IP address. It covers the endpoints anyone can call before the bank has granted them anything. Trip code: `OBP-10060`. + |2. **Authentication limiter** (`auth.rate_limit.*`) runs inside the credential check of Direct Login, DAuth, Gateway Login and SIWE, before the password or token is verified, keyed by IP address and by account. It defends against brute force, credential stuffing and lockout attacks. Trip code: `OBP-10061`. + |3. **Consumer quota** (the limits described above) runs after authentication, keyed by Consumer, or by IP address with a single hourly ceiling for anonymous calls. It is the commercial and fair-use quota. Trip code: `OBP-10018`. + | + |A login attempt is counted by the authentication limiter only; it is not a self-service scope, so no attempt is counted twice. Every limiter counts in Redis and fails open: a Redis outage never blocks a call. + | + |### Self-service rate limiting (per IP address, before any credential) + | + |The limits above are keyed by Consumer, so they cannot protect the calls a client makes before it has one. Those endpoints are covered by the self-service limiter, keyed by the client IP address, grouped in scopes: + | + |- **signup** — Create User (self-registration), Validate User Email, Get User Invitation Information + |- **password_reset** — Request Password Reset Email, Complete Password Reset + |- **consent_request** — Create Consent Request, Create Consent Request VRP + |- **consumer_registration** — Create a Consumer (Dynamic Registration) + |- **lookup** — Validate and check IBAN + |- **signal_channel_create** — Publish Signal Message, counted only when it creates a new channel (over gRPC this scope is keyed by Consumer instead) + | + |Each scope has per-minute, per-hour and per-day limits per IP, with built-in defaults chosen so that a person or a well-behaved agent never reaches them, plus an optional global per-hour cap across all addresses that acts as a circuit breaker. Every request is counted, whether or not it succeeds. Counters live in Redis and fail open. + | + |**Shadow mode (the default).** The limiter is on out of the box but does not block. A request over a limit is logged once per window (`event=self_service_rate_limit_shadow_trip`) and the response carries: + | + | X-Rate-Limit-Warning: OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate limit for signup (5 per hour) in the future. + | + |No enforcement date is claimed unless the operator sets `self_service.rate_limit.enforce_announced_from`, in which case ", from " is appended. Every self-service response also carries `X-Rate-Limit-Limit`, `X-Rate-Limit-Remaining` and `X-Rate-Limit-Reset` for the window the caller is closest to exhausting, so a client can back off before enforcement starts. + | + |**Enforce mode.** Set `self_service.rate_limit.mode = enforce` and a trip answers **429** with `OBP-10060`, a `Retry-After` header and the same `X-Rate-Limit-*` headers, without running the endpoint. + | + |Limits are set with `self_service.rate_limit..per_ip.per_minute|per_hour|per_day`, `self_service.rate_limit..global.per_hour`, or the generic `self_service.rate_limit.per_ip.*`; -1 switches a window off and 0 blocks it. See the props template for the built-in numbers. Behind a proxy, configure `trust.proxy.enabled` and `trust.proxy.header` so the client address is the real one; otherwise every caller shares the proxy's counters. """) glossaryItems += GlossaryItem( @@ -3728,7 +3761,7 @@ object Glossary extends MdcLoggable { | |A helper endpoint (`POST /management/dynamic-resource-docs/endpoint-code`) can generate a method-body template from example request / response bodies. | -|See ${getGlossaryItemLink("Dynamic Code Paths")} for how Dynamic Resource Docs relate to the other runtime-defined building blocks. +|See ${getGlossaryItemLink("Dynamic Code Paths")} for how Dynamic Resource Docs relate to the other runtime-defined building blocks, and ${getGlossaryItemLink("Dynamic Change Request")} for how an operator can require a second person to approve each definition before it is compiled and served. | """.stripMargin) @@ -3789,6 +3822,60 @@ object Glossary extends MdcLoggable { | |Runtime-compiled code (Dynamic Resource Docs, Connector Methods, Dynamic Message Docs) is disabled unless the `allow_user_generated_scala_code` prop is set to true, and every creation endpoint requires its corresponding Role. Dynamic Endpoints (swagger, no code) are not affected by that prop; each generated endpoint is protected by its own auto-generated Role. | +""".stripMargin) + + glossaryItems += GlossaryItem( + title = "Dynamic Change Request", + description = + s""" +|A Dynamic Change Request is a proposed create, update or delete of a runtime-defined artefact that carries code or configuration - a ${getGlossaryItemLink("Dynamic Resource Doc")}, a ${getGlossaryItemLink("Dynamic Message Doc")}, a ${getGlossaryItemLink("Connector Method")} or an ABAC Rule - held for approval by a second person. It is how OBP implements *maker/checker* for dynamic code. +| +|**Why** +| +|A Dynamic Resource Doc method body, a Connector Method or a Dynamic Message Doc is user-supplied code compiled and run inside the OBP-API JVM, with the connector credentials and reach of the whole instance. The sandbox is not a meaningful second line of defence, so the primary control is that the person who writes the code (the *maker*) can never make it live alone: a different User holding the Role `CanApproveDynamicChangeRequest` (the *checker*) reviews the exact definition and approves it. +| +|**How it works when approval is on** +| +|1) The maker calls the usual v4.0.0 / v6.0.0 create, update or delete endpoint. OBP checks the maker's Role, validates the JSON and compiles the code exactly as before, but instead of applying the change it stores a Dynamic Change Request and answers `202 Accepted` with the request instead of the artefact. Nothing is served yet. +| +|2) The checker reads the request (`GET /obp/v7.0.0/management/dynamic-change-requests/CHANGE_REQUEST_ID`, which returns the proposed and the current payload side by side) and approves it by quoting its `payload_hash`, the SHA-256 of the exact body, on `POST .../approval`. Only then is the change applied. OBP refuses an approval from the User who made the request (`OBP-30279`). +| +|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose body hash equals the hash a checker approved, so a row edited directly in the database does not run. +| +|4) Deactivating an artefact is a direct action by a single checker (`POST .../deactivation`), with no request: four eyes to enable, one pair to disable. Enabling it again goes through a request. +| +|Approval is system level. Dynamic code runs in the shared JVM, so a bank-level artefact is approved by the same system-level checker; there are no bank-level change request endpoints. +| +|**Statuses** +| +|* `INITIATED` - waiting for a checker. Approve, reject and withdraw apply only in this status. +|* `APPROVED` - applied; the artefact is compiled and served. +|* `REJECTED` - declined by a checker with a comment. Nothing was applied. +|* `WITHDRAWN` - taken back by the maker. +|* `EXPIRED` - no checker acted within the request time-to-live. +|* `FAILED` - approved, but applying the change threw. The error is recorded on the request and nothing half-applied is served. +| +|**Endpoints (v7.0.0, tag Dynamic-Change-Request)** +| +|* `GET /management/dynamic-code-approval-config` - whether approval is on, for which target types, and the request time-to-live. Any authenticated User; what a client reads to warn a maker before they submit. +|* `GET /my/dynamic-change-requests` - the caller's own requests. No Role. +|* `GET /management/dynamic-change-requests` and `GET .../CHANGE_REQUEST_ID` - all requests, filterable by `status`, `target_type`, `target_id` and `requestor_user_id`. Role `CanGetDynamicChangeRequests`. +|* `POST /management/dynamic-change-requests` - submit a request explicitly with a `business_justification`, for tooling; the maker's usual create call does this implicitly. +|* `POST .../CHANGE_REQUEST_ID/approval`, `.../rejection` (comment required), `.../withdrawal` (maker only). +|* `POST /management/dynamic-resource-docs/ID/deactivation` and the equivalents for dynamic message docs, connector methods and ABAC rules. +| +|**Operator settings (props)** +| +|* `allow_user_generated_scala_code` - the kill switch for all runtime-compiled code. Default false: every create or update of dynamic code fails with `OBP-50020` and nothing dynamic runs, whatever the settings below say. Read once at startup, so changing it needs a restart. +|* `dynamic_code_requires_approval` - the approval switch. Default false: writes go live immediately, today's behaviour. +|* `dynamic_code_approval_target_types` - which target types are gated. Default `DYNAMIC_RESOURCE_DOC,DYNAMIC_MESSAGE_DOC,CONNECTOR_METHOD,ABAC_RULE`. +|* `dynamic_code_delete_requires_approval` - whether deletes are queued too. Default true; deleting does not expand capability but does break consumers. +|* `dynamic_code_approval_request_ttl_hours` - `INITIATED` requests older than this become `EXPIRED` when next read. 0 disables expiry. Default 168. +| +|The first start with `dynamic_code_requires_approval=true` seeds the approved hash of every pre-existing row from its current body, once per database, so nothing that is live today stops working. After that the only way a row becomes executable is a checker's approval. +| +|See ${getGlossaryItemLink("Dynamic Code Paths")} for how the gated artefacts relate to each other. +| """.stripMargin) glossaryItems += GlossaryItem( @@ -6041,7 +6128,7 @@ object Glossary extends MdcLoggable { |Not to be confused with [Chat](/glossary#Chat), which is the persistent, human-facing messaging surface (rooms, threads, reactions, read markers). | |## Lifecycle - |- Channels are auto-created on first publish; no registration step. + |- Channels are auto-created on first publish; no registration step. Creating channels is rate limited per caller (scope `signal_channel_create`, see [Rate Limiting](/glossary#Rate-Limiting)); publishing to an existing channel is not. |- On this instance a channel expires ${code.api.cache.RedisMessaging.channelTtlSeconds} seconds after its last publish, and holds at most ${code.api.cache.RedisMessaging.channelMaxMessages} messages (oldest are trimmed). |- Channel names are 1 to 128 characters from letters, digits, dot, underscore and hyphen. |- Every message carries a per-channel monotonic **sequence** (Redis server time in microseconds, forced strictly increasing) stamped atomically when it is stored. Poll with `after_sequence=` and continue from the response's `next_after_sequence`. Do not poll by offset: trimming moves list positions, so an offset-tracking poller silently skips messages once the channel is full. Sequences are time-based rather than a counter so a cursor stays valid across a channel expiring and being recreated. @@ -6066,11 +6153,49 @@ object Glossary extends MdcLoggable { |## Payloads are data, not instructions |Signal channels are readable and writable by any authenticated consumer on the instance. If your agent feeds received payloads to an LLM, treat them as **untrusted data, never as instructions** — the character checks above stop display-layer trickery, but no server-side check can stop a payload from *saying* something misleading. Prompt-injection defence belongs in the consuming agent. | + |## Getting credentials as an agent (no account needed) + |An agent does not need a pre-existing OBP user, consumer key or password. Where the instance runs OBP-OIDC with dynamic client registration enabled, three unauthenticated calls are enough: + | + |1. **Discover the identity provider.** `GET /obp/v6.0.0/well-known` lists the OpenID discovery documents this instance trusts. Fetch the `obp-oidc` one; its `registration_endpoint` and `token_endpoint` are the two URLs used below. + |2. **Register a client** (RFC 7591 dynamic client registration; no initial access token is required): + | + | curl -X POST REGISTRATION_ENDPOINT -H "Content-Type: application/json" -d '{"client_name":"my-agent","grant_types":["client_credentials"],"token_endpoint_auth_method":"client_secret_post","redirect_uris":["http://localhost/unused"]}' + | + | The response carries `client_id` and `client_secret`. Store them; the secret is shown once. Behind the scenes OBP-OIDC also creates the matching OBP Consumer, so `client_id` is the consumer key. + |3. **Get a token** with the client credentials grant (no user, no browser): + | + | curl -X POST TOKEN_ENDPOINT -d "grant_type=client_credentials&client_id=YOUR_CLIENT_ID&client_secret=YOUR_CLIENT_SECRET&scope=openid" + | + |4. **Call OBP** with `Authorization: Bearer YOUR_ACCESS_TOKEN`, on REST or gRPC. + | + |What OBP does with such a token: it recognises the OBP-OIDC issuer, resolves the Consumer from the token's `azp` claim, and creates (once) a User whose provider id is the client id. `GET /obp/v6.0.0/users/current` and `GET /obp/v7.0.0/consumers/current/identity` show the resulting identity. That User starts with **no entitlements**, so it can list channels, read channel info, fetch and publish messages, and receive private messages addressed to its user id — but it cannot call Get Signal Channel Stats or Delete Signal Channel, and it has no access to any bank data. Every message it publishes carries its consumer id and user id, so an agent registered this way is attributable and its Consumer can be disabled by an operator. + | + |Do **not** use `POST /obp/v6.0.0/dynamic-registration/consumers` for this. Despite the similar name it is the PSD2 path: it needs a QWAC certificate matching a pre-registered Regulated Entity and is meant for regulated third-party providers, not agents. + | + |Operators: because registration is unauthenticated, expose it only with rate limiting on registrations per IP and in total, or require an initial access token in production. See the OBP-OIDC README. + | |## Endpoints |See the API Explorer tags **Signal-Channel** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal-channels/...`. | + |Note on `total_count` in Get Signal Messages: it counts every message in the channel, including private messages hidden from the caller, so it can exceed the number of messages returned. Poll with `after_sequence` and `next_after_sequence` rather than comparing counts. + | |## gRPC - |The same operations are served over gRPC by `SignalChannelsService` (package `code.obp.grpc.signal.g1`, contract in `signal.proto`) when the gRPC server is enabled (`grpc.server.enabled`): **Publish**, **Fetch** and **ListChannels** are 1:1 with the REST endpoints and share their storage, and **Subscribe** is a server-side stream of new messages on one channel. Subscribe is live only — no catch-up, no replay — and applies the same privacy filter as Fetch. Each publish, REST or gRPC, is pushed to subscribers through Redis pub/sub. Authenticate with the same `Authorization` value the REST endpoints take, sent as gRPC metadata. + |The same operations are served over gRPC by `SignalChannelsService` (package `code.obp.grpc.signal.g1`, contract in `signal.proto`) when the gRPC server is enabled (`grpc.server.enabled`): **Publish**, **Fetch** and **ListChannels** are 1:1 with the REST endpoints and share their storage, and **Subscribe** is a server-side stream of new messages on one channel. Subscribe is live only — no catch-up, no replay — and applies the same privacy filter as Fetch. Each publish, REST or gRPC, is pushed to subscribers through Redis pub/sub. + | + |**Authentication.** Send the same value the REST `Authorization` header takes (`Bearer YOUR_ACCESS_TOKEN` or `DirectLogin token=YOUR_TOKEN`) as gRPC metadata under the key `authorization`. A call without it fails with status UNAUTHENTICATED and the message "Missing authorization header". + | + |**Discovery.** The server exposes gRPC reflection, so generic clients can list and describe the service without the proto file. With grpcurl (plaintext shown; use TLS as your deployment requires): + | + | grpcurl -plaintext HOST:PORT list + | grpcurl -plaintext HOST:PORT describe code.obp.grpc.signal.g1.SignalChannelsService + | grpcurl -plaintext -H "authorization: Bearer YOUR_ACCESS_TOKEN" HOST:PORT code.obp.grpc.signal.g1.SignalChannelsService/ListChannels + | grpcurl -plaintext -H "authorization: Bearer YOUR_ACCESS_TOKEN" -d '{"channel_name":"discovery","after_sequence":0,"limit":50}' HOST:PORT code.obp.grpc.signal.g1.SignalChannelsService/Fetch + | grpcurl -plaintext -H "authorization: Bearer YOUR_ACCESS_TOKEN" -d @ HOST:PORT code.obp.grpc.signal.g1.SignalChannelsService/Publish < publish.json + | grpcurl -plaintext -H "authorization: Bearer YOUR_ACCESS_TOKEN" -d '{"channel_name":"discovery"}' HOST:PORT code.obp.grpc.signal.g1.SignalChannelsService/Subscribe + | + |where publish.json holds the request with the payload as an escaped JSON string, for example `{"channel_name":"discovery","message_type":"announce","payload_json":"{ ... your JSON, with its inner quotes escaped ... }"}`. HOST:PORT is the gRPC listener, port `grpc.server.port` (default 50051), separate from the HTTP port. + | + |Over gRPC the payload travels as `payload_json`, a string holding the JSON-encoded payload verbatim (protobuf has no native JSON value type). Int64 fields such as `sequence` and `message_count` arrive as strings in JSON-transcoded output; that is standard protobuf JSON mapping, not a change of type. | """) diff --git a/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala new file mode 100644 index 0000000000..ff8e6024c1 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala @@ -0,0 +1,212 @@ +package code.api.util + +import code.api.Constant.CALL_COUNTER_PREFIX +import code.api.util.ErrorMessages.{RateLimitFutureWarning, TooManyRequestsSelfService} +import code.api.util.RateLimitingPeriod._ +import code.util.Helper.MdcLoggable +import net.liftweb.common.Full + +/** Rate limiter for the self-service endpoints: the calls a client can make before it holds + * any credential, or with credentials it obtained for free (sign-up, password reset, + * consent requests, consumer registration, lookups, signal channel creation). + * + * One of three limiters, each with its own 429 code: + * - [[RateLimitingUtil]] post-auth, keyed by Consumer, the commercial quota -> OBP-10018 + * - this limiter pre-auth, keyed by client IP, abuse of the free tier -> OBP-10060 + * - [[AuthRateLimiter]] inside the credential check, keyed by IP and account -> OBP-10061 + * Login attempts are NOT a self-service scope: AuthRateLimiter owns every credential check + * (DirectLogin via AuthUser.getResourceUserId, DAuth, GatewayLogin, SIWE), so counting them + * here too would count each attempt twice. Every self-service endpoint belongs to a + * named *scope*; each scope has per-key limits (per minute, per hour, per day, keyed by the + * client IP address, or by consumer for gRPC) and an optional global per-hour cap that acts + * as a circuit breaker against a spray from many addresses. + * + * Props (all optional; built-in defaults apply): + * - `self_service.rate_limit.enabled` (default true) + * - `self_service.rate_limit.mode` = shadow (default) | enforce + * - `self_service.rate_limit.per_ip.per_minute|per_hour|per_day` generic per-key limits + * - `self_service.rate_limit..per_ip.per_minute|per_hour|per_day` per-scope overrides + * - `self_service.rate_limit..global.per_hour` per-scope global cap (-1 = off) + * - `self_service.rate_limit.enforce_announced_from` free text appended to the warning + * A limit of -1 disables that window; 0 blocks every call in that window. + * + * Shadow mode: trips are logged and reported to the caller through the + * `X-Rate-Limit-Warning` header (see [[warningMessage]]), but the request is allowed. + * Enforce mode: trips produce [[Blocked]] and the caller renders 429. + * + * Counting is attempt-based (every request increments) and fail-open: if Redis is + * unavailable, [[RateLimitingUtil.incrementCounter]] returns (-1, -1) and the window is + * skipped, so a Redis outage never blocks a self-service call. + */ +object SelfServiceRateLimiter extends MdcLoggable { + + val PropsPrefix = "self_service.rate_limit" + + val ModeShadow = "shadow" + val ModeEnforce = "enforce" + + /** Built-in limits for one scope. */ + final case class ScopeDefaults(perMinute: Long, perHour: Long, perDay: Long, globalPerHour: Long) + + /** Used for any scope that has no entry in [[scopeDefaults]]. */ + val genericDefaults: ScopeDefaults = ScopeDefaults(perMinute = 10, perHour = 60, perDay = 200, globalPerHour = -1) + + /** Sensible defaults per scope. A real person or a well-behaved agent never gets near these + * from one address; they are meant to be hit only by scripts. */ + val scopeDefaults: Map[String, ScopeDefaults] = Map( + "signup" -> ScopeDefaults(perMinute = 3, perHour = 5, perDay = 10, globalPerHour = 500), + "password_reset" -> ScopeDefaults(perMinute = 3, perHour = 5, perDay = 10, globalPerHour = 500), + "consent_request" -> ScopeDefaults(perMinute = 10, perHour = 30, perDay = 100, globalPerHour = -1), + "consumer_registration" -> ScopeDefaults(perMinute = 5, perHour = 10, perDay = 20, globalPerHour = 500), + "lookup" -> ScopeDefaults(perMinute = 20, perHour = 60, perDay = 200, globalPerHour = -1), + "signal_channel_create" -> ScopeDefaults(perMinute = 5, perHour = 20, perDay = 50, globalPerHour = -1) + ) + + /** One counter window after this request was counted. */ + final case class Window(name: String, period: LimitCallPeriod, limit: Long, current: Long, resetSeconds: Long) { + def remaining: Long = math.max(0L, limit - current) + /** limit > 0 and the count is over it. A limit of 0 blocks every call (current is always >= 1). */ + def exceeded: Boolean = limit >= 0 && current > limit + def describe: String = s"$limit ${RateLimitingPeriod.humanReadable(period)}" + } + + sealed trait Outcome { + def scope: String + def windows: List[Window] + /** The window the caller is closest to exhausting: fewest remaining calls, then soonest reset. + * This is what the X-Rate-Limit-* headers describe. */ + def tightest: Option[Window] = + windows.filter(_.limit >= 0).sortBy(w => (w.remaining, w.resetSeconds)).headOption + def exceededWindow: Option[Window] = None + } + /** Limiter disabled, or Redis unavailable for every window: nothing was counted. */ + final case class Skipped(scope: String) extends Outcome { val windows: List[Window] = Nil } + final case class Allowed(scope: String, windows: List[Window]) extends Outcome + /** A limit was exceeded but the mode is shadow: the request proceeds with a warning. */ + final case class Warned(scope: String, windows: List[Window], exceeded: Window) extends Outcome { + override def exceededWindow: Option[Window] = Some(exceeded) + } + /** A limit was exceeded and the mode is enforce: the caller must respond 429. */ + final case class Blocked(scope: String, windows: List[Window], exceeded: Window) extends Outcome { + override def exceededWindow: Option[Window] = Some(exceeded) + } + + def enabled: Boolean = APIUtil.getPropsAsBoolValue(s"$PropsPrefix.enabled", true) + + def mode: String = APIUtil.getPropsValue(s"$PropsPrefix.mode", ModeShadow).trim.toLowerCase match { + case ModeEnforce => ModeEnforce + case _ => ModeShadow + } + + def isEnforcing: Boolean = mode == ModeEnforce + + /** Optional operator text naming when enforcement is planned, e.g. "2026-10-01". Only + * appended to the warning when set; nothing about timing is claimed otherwise. */ + def enforceAnnouncedFrom: Option[String] = + APIUtil.getPropsValue(s"$PropsPrefix.enforce_announced_from").toOption.map(_.trim).filter(_.nonEmpty) + + /** Resolution order for a per-key limit: explicit scope prop, explicit generic prop, + * built-in scope default, built-in generic default. */ + def perKeyLimit(scope: String, dimension: String): Long = { + val builtIn = scopeDefaults.getOrElse(scope, genericDefaults) + val builtInValue = dimension match { + case "per_minute" => builtIn.perMinute + case "per_hour" => builtIn.perHour + case "per_day" => builtIn.perDay + case _ => -1L + } + APIUtil.getPropsAsLongValue(s"$PropsPrefix.$scope.per_ip.$dimension") match { + case Full(v) => v + case _ => APIUtil.getPropsAsLongValue(s"$PropsPrefix.per_ip.$dimension") match { + case Full(v) => v + case _ => builtInValue + } + } + } + + def globalPerHourLimit(scope: String): Long = + APIUtil.getPropsAsLongValue(s"$PropsPrefix.$scope.global.per_hour") match { + case Full(v) => v + case _ => scopeDefaults.getOrElse(scope, genericDefaults).globalPerHour + } + + /** Count this request against `scope` for `key` and report the outcome. + * + * @param scope the endpoint class, e.g. "signup" + * @param key what the per-key windows are keyed on; normally the client IP address + * @param keyKind label for logs and keys, "ip" (default) or "consumer" + */ + def check(scope: String, key: String, keyKind: String = "ip"): Outcome = { + if (!enabled) return Skipped(scope) + + val safeScope = sanitise(scope) + val safeKey = if (key == null || key.trim.isEmpty || key.equalsIgnoreCase("unknown")) "" else sanitise(key.trim) + + val perKeyWindows: List[(String, LimitCallPeriod, Long)] = + if (safeKey.isEmpty) Nil // no usable key: only the global window can be checked + else List( + (s"${keyKind}_per_minute", PER_MINUTE, perKeyLimit(safeScope, "per_minute")), + (s"${keyKind}_per_hour", PER_HOUR, perKeyLimit(safeScope, "per_hour")), + (s"${keyKind}_per_day", PER_DAY, perKeyLimit(safeScope, "per_day")) + ) + val globalWindow: List[(String, LimitCallPeriod, Long)] = + List(("global_per_hour", PER_HOUR, globalPerHourLimit(safeScope))) + + val windows: List[Window] = (perKeyWindows ++ globalWindow).flatMap { case (name, period, limit) => + if (limit < 0) None // -1: this window is switched off, do not touch Redis + else { + val redisKey = + if (name.startsWith("global")) buildKey(safeScope, "global", period) + else buildKey(safeScope, s"${keyKind}_$safeKey", period) + val (ttl, current) = RateLimitingUtil.incrementCounter(redisKey, period) + // current == -1 signals Redis unavailable: fail open by dropping this window. + if (current < 0) None else Some(Window(name, period, limit, current, ttl)) + } + } + + if (windows.isEmpty) return Skipped(safeScope) + + // Report the shortest exceeded window first, matching the consumer limiter's precedence. + windows.find(_.exceeded) match { + case None => Allowed(safeScope, windows) + case Some(exceeded) if isEnforcing => + logger.warn(logLine("trip", safeScope, keyKind, safeKey, exceeded)) + Blocked(safeScope, windows, exceeded) + case Some(exceeded) => + // Shadow: the first request over the limit in a window is warn, the rest debug, so a + // burst produces one line per window rather than one per request. + val line = logLine("shadow_trip", safeScope, keyKind, safeKey, exceeded) + if (exceeded.current == exceeded.limit + 1) logger.warn(line) else logger.debug(line) + Warned(safeScope, windows, exceeded) + } + } + + /** The `X-Rate-Limit-Warning` text for a shadow trip: + * "OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate + * limit for signup (5 per hour) in the future." plus ", from " when the operator has + * announced one. */ + def warningMessage(scope: String, exceeded: Window): String = { + val base = RateLimitFutureWarning + .replace("SCOPE", scope) + .replace("LIMIT", exceeded.describe) + enforceAnnouncedFrom match { + case Some(from) => base.stripSuffix(".") + s", from $from." + case None => base + } + } + + /** The 429 body text for an enforced trip. OBP-10060, so a client can tell this limiter from the + * Consumer quota (OBP-10018) and the authentication limiter (OBP-10061). */ + def blockedMessage(scope: String, exceeded: Window): String = + s"$TooManyRequestsSelfService The rate limit for $scope is ${exceeded.describe}. Try again in ${exceeded.resetSeconds} seconds." + + private def buildKey(scope: String, subject: String, period: LimitCallPeriod): String = + s"${CALL_COUNTER_PREFIX}self_service_${scope}_${subject}_${RateLimitingPeriod.toString(period)}" + + /** Keep Redis keys and log lines free of separators and whitespace. */ + private def sanitise(s: String): String = s.replaceAll("[^A-Za-z0-9._:\\-]", "_") + + private def logLine(event: String, scope: String, keyKind: String, key: String, w: Window): String = + s"event=self_service_rate_limit_$event scope=$scope key_kind=$keyKind key=${if (key.isEmpty) "none" else key} " + + s"counter=${w.name} current=${w.current} limit=${w.limit} retry_after_s=${w.resetSeconds}" +} diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala index 2bd61744cc..8867f8911f 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala @@ -169,7 +169,11 @@ object Http4sApp extends MdcLoggable { // least obvious, a proxy forwarding the header over a hop with no client certificate, enables // no TLS middleware at all, so neither step can live in Http4sServer's mtls.enabled branch. val req = CallerCertificate.resolveCaller(Psd2CertIngress.canonicalize(rawReq)) - app.run(req) + // Self-service rate limiting (sign-up, password reset, consent requests, consumer + // registration, lookups, signal channel creation) runs here, before routing, keyed by the + // client IP. In shadow mode it only adds X-Rate-Limit-* headers; in enforce mode a trip + // answers 429 without running the route. See SelfServiceRateLimitMiddleware. + SelfServiceRateLimitMiddleware(req)(app.run) .map(resp => stripBodyForHead(req, Http4sStandardHeaders(req, resp))) .handleErrorWith { e => logger.error(s"[Http4sApp] Uncaught exception: ${req.method} ${req.uri} - ${e.getMessage}", e) diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index 3c61134aff..15faa0f102 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -616,6 +616,10 @@ object Http4sCallContextBuilder { * Extract the trusted client IP. Defaults to the immediate socket peer; consults a * forwarded-for header only when `trust.proxy.enabled = true`. See [[RemoteIpUtil]]. */ + /** The trusted client IP for a request, as CallContext.ipAddress would carry it. Public so + * request-level middleware (SelfServiceRateLimitMiddleware) keys on the same value. */ + def clientIp(request: Request[IO]): String = extractIpAddress(request) + private def extractIpAddress(request: Request[IO]): String = { val socketPeer = request.remoteAddr.map(_.toUriString).getOrElse("") RemoteIpUtil.resolveClientIp( diff --git a/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala new file mode 100644 index 0000000000..74821e42eb --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala @@ -0,0 +1,121 @@ +package code.api.util.http4s + +import cats.effect.IO +import code.api.util.SelfServiceRateLimiter +import code.api.util.SelfServiceRateLimiter.{Blocked, Outcome, Skipped, Warned, Window} +import code.util.Helper.MdcLoggable +import org.http4s.{Header, Headers, Method, Request, Response, Status} +import org.typelevel.ci.CIString + +import scala.util.matching.Regex + +/** Applies [[SelfServiceRateLimiter]] to the self-service endpoints by path, before routing. + * + * Sits in Http4sApp.httpApp around the whole route chain, so it needs nothing from the + * endpoint: the scope comes from a (method, path) table and the key is the client IP address + * resolved the same way CallContext.ipAddress is. Runs for every API version because the + * patterns are version-agnostic. + * + * Per request, when a scope matches: + * - the request is counted; + * - enforce mode and over a limit: 429 with the OBP error body, `Retry-After` and the + * `X-Rate-Limit-*` headers, and the route is never run; + * - otherwise the route runs and the response gains `X-Rate-Limit-Limit`, `-Remaining` + * and `-Reset` for the tightest window (only when the response has none already) plus, + * on a shadow trip, `X-Rate-Limit-Warning`. + */ +object SelfServiceRateLimitMiddleware extends MdcLoggable { + + val WarningHeader = "X-Rate-Limit-Warning" + val LimitHeader = "X-Rate-Limit-Limit" + val RemainingHeader = "X-Rate-Limit-Remaining" + val ResetHeader = "X-Rate-Limit-Reset" + + /** A self-service endpoint class. `condition` lets an entry opt out per request, e.g. a + * signal publish only counts as channel creation when the channel does not exist yet. */ + final case class Entry(scope: String, method: Method, path: Regex, condition: (Request[IO], Regex.Match) => Boolean = (_, _) => true) + + private val V = "/obp/v[^/]+" // any /obp/vN.N.N prefix + + /** The self-service table. Order matters only for readability; the first match wins. */ + val entries: List[Entry] = List( + // Logins are deliberately absent: AuthRateLimiter counts every credential check (DirectLogin, + // DAuth, GatewayLogin, SIWE) by IP and by account, so a login entry here would count twice. + // signup: self-registration and the tokens it emails + Entry("signup", Method.POST, s"^$V/users$$".r), + Entry("signup", Method.POST, s"^$V/users/email-validation$$".r), + Entry("signup", Method.POST, s"^$V/banks/[^/]+/user-invitations$$".r), + // password_reset: mail sending and token guessing + Entry("password_reset", Method.POST, s"^$V/users/password-reset-url$$".r), + Entry("password_reset", Method.POST, s"^$V/users/password$$".r), + // consent_request: anonymous rows created on behalf of a TPP + Entry("consent_request", Method.POST, s"^$V/consumer/consent-requests$$".r), + Entry("consent_request", Method.POST, s"^$V/consumer/vrp-consent-requests$$".r), + // consumer_registration: each success creates a Consumer + Entry("consumer_registration", Method.POST, s"^$V/dynamic-registration/consumers$$".r), + // lookup: read-only but reaches a connector + Entry("lookup", Method.POST, s"^$V/account/check/scheme/iban$$".r), + // signal_channel_create: the one unbounded write into Redis. Counted only when the + // channel named in the path does not exist yet, so ordinary publishing is untouched. + Entry("signal_channel_create", Method.POST, s"^$V/signal-channels/([^/]+)/messages$$".r, + (_, m) => code.api.cache.RedisMessaging.channelInfo(m.group(1)).isEmpty) + ) + + def scopeFor(req: Request[IO]): Option[String] = { + val path = req.uri.path.renderString + entries.iterator.map { e => + if (e.method != req.method) None + else e.path.findFirstMatchIn(path).filter(m => safely(e.condition(req, m))).map(_ => e.scope) + }.collectFirst { case Some(scope) => scope } + } + + private def safely(b: => Boolean): Boolean = + try b catch { case scala.util.control.NonFatal(e) => + logger.warn(s"SelfServiceRateLimitMiddleware condition failed open: ${e.getMessage}") + false + } + + /** Wrap the application. */ + def apply(req: Request[IO])(run: Request[IO] => IO[Response[IO]]): IO[Response[IO]] = + scopeFor(req) match { + case None => run(req) + case Some(scope) => + IO.blocking(SelfServiceRateLimiter.check(scope, Http4sCallContextBuilder.clientIp(req), "ip")).flatMap { + case Blocked(s, _, exceeded) => IO.pure(blockedResponse(s, exceeded)) + case outcome => run(req).map(resp => decorate(resp, outcome)) + } + } + + private def blockedResponse(scope: String, exceeded: Window): Response[IO] = { + val message = SelfServiceRateLimiter.blockedMessage(scope, exceeded) + val escaped = message.replace("\\", "\\\\").replace("\"", "\\\"") + Response[IO](status = Status.TooManyRequests) + .withEntity(s"""{"code":429,"message":"$escaped"}""".getBytes("UTF-8")) + .withHeaders(Headers( + Header.Raw(CIString("Content-Type"), "application/json; charset=utf-8"), + Header.Raw(CIString("Retry-After"), exceeded.resetSeconds.toString), + Header.Raw(CIString(LimitHeader), exceeded.limit.toString), + Header.Raw(CIString(RemainingHeader), "0"), + Header.Raw(CIString(ResetHeader), exceeded.resetSeconds.toString) + )) + } + + private def decorate(resp: Response[IO], outcome: Outcome): Response[IO] = outcome match { + case Skipped(_) => resp + case _ => + val hasLimitHeaders = resp.headers.headers.exists(_.name.toString.equalsIgnoreCase(LimitHeader)) + val counterHeaders: List[Header.Raw] = outcome.tightest.toList.filterNot(_ => hasLimitHeaders).flatMap { w => + List( + Header.Raw(CIString(LimitHeader), w.limit.toString), + Header.Raw(CIString(RemainingHeader), w.remaining.toString), + Header.Raw(CIString(ResetHeader), w.resetSeconds.toString) + ) + } + val warning: List[Header.Raw] = outcome match { + case Warned(scope, _, exceeded) => + List(Header.Raw(CIString(WarningHeader), SelfServiceRateLimiter.warningMessage(scope, exceeded))) + case _ => Nil + } + (counterHeaders ++ warning).foldLeft(resp)((r, h) => r.putHeaders(h)) + } +} diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 9d6a7780b2..fc65fa3ce8 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -3131,6 +3131,11 @@ object Http4s510 { | |This endpoint provides **secure, validated consumer registration** unlike the standard `/management/consumers` endpoint. | + |**Not for AI agents or ordinary applications.** This is the PSD2 path and requires a QWAC certificate that matches a + |pre-registered Regulated Entity. An agent or app that simply needs credentials should use OAuth2 dynamic client + |registration (RFC 7591) on the OBP-OIDC identity provider instead; see the glossary entry "Signal Channels", + |section "Getting credentials as an agent". + | |**How it works (for comprehension flow):** | |1. **Extract JWT from request**: Parse the signed JWT from the request body diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 00912c3c4c..5339090275 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -10433,6 +10433,14 @@ object Http4s600 { |Only channels that contain at least one broadcast message (no to_user_id) are listed. |Private-only channels are not shown. | + |**Getting credentials as an agent.** No pre-existing OBP account is needed. Register an OAuth2 client + |with the OBP-OIDC dynamic client registration endpoint (RFC 7591; its URL is the registration_endpoint + |of the OpenID discovery document listed by GET /obp/v6.0.0/well-known), exchange the returned client_id + |and client_secret for an access token with grant_type=client_credentials, and send it as + |`Authorization: Bearer `. OBP creates a Consumer and a User for the client automatically; that + |identity has no Roles, which is enough for this endpoint. The walkthrough is in the glossary under + |"Signal Channels". (POST /obp/v6.0.0/dynamic-registration/consumers is the PSD2 certificate path, not this.) + | |Authentication is Required. | |""".stripMargin, @@ -10515,6 +10523,16 @@ object Http4s600 { |Live delivery: every publish is also pushed to gRPC clients streaming the channel via |SignalChannelsService.Subscribe (see signal.proto). The same service offers Publish, Fetch |and ListChannels as 1:1 equivalents of the REST endpoints, over the same Redis storage. + |gRPC calls authenticate with the same Authorization value sent as metadata under the key + |`authorization`; the server supports reflection, so grpcurl can discover the service. + | + |**Getting credentials as an agent.** No pre-existing OBP account is needed. Register an OAuth2 client + |with the OBP-OIDC dynamic client registration endpoint (RFC 7591; its URL is the registration_endpoint + |of the OpenID discovery document listed by GET /obp/v6.0.0/well-known), exchange the returned client_id + |and client_secret for an access token with grant_type=client_credentials, and send it as + |`Authorization: Bearer `. OBP creates a Consumer and a User for the client automatically; that + |identity has no Roles, which is enough for this endpoint. The walkthrough is in the glossary under + |"Signal Channels". (POST /obp/v6.0.0/dynamic-registration/consumers is the PSD2 certificate path, not this.) | |Authentication is Required. | @@ -10563,6 +10581,10 @@ object Http4s600 { | |Without `after_sequence`, offset and limit page over the channel's current contents. | + |`total_count` is the number of messages in the channel including private messages hidden + |from you, so it can be larger than the number of messages returned. Do not use it to detect + |missed messages; use `after_sequence` and `next_after_sequence`. + | |Authentication is Required. | |""".stripMargin, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 5e30dab65e..8d678f16b1 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -8,7 +8,7 @@ import code.api.Constant._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._ import code.api.util.APIUtil.{EmptyBody, _} import code.api.util.{APIUtil, ApiRole, CallContext, CustomJsonFormats, Glossary, NewStyle} -import code.api.util.ApiRole.{canAttachOpenCorridorPromise, canConfigureAmqpBankBroker, canGetMessageOutbox, canRetryMessageOutbox, canSettleOpenCorridor, canCreateAccount, canCreateEntitlementAtAnyBank, canCreateEntitlementAtOneBank, canCreateMetricsArchiveRun, canCreateOrganisation, canCreateRoutingScheme, canCreateTestEmail, canCreateUtilityVendResult, canDeleteEntitlementAtAnyBank, canDeleteOrganisation, canDeleteRoutingScheme, canDeleteSchedulerJobLock, canGetAccountAccessTrace, canGetAnyOrganisation, canGetAnyUser, canGetCacheConfig, canGetCacheInfo, canGetCacheNamespaces, canGetConnectorHealth, canGetCustomersAtOneBank, canGetDatabasePoolInfo, canGetMetricsDiagnostics, canGetMigrations, canGetSchedulerJobLocks, canReadMetrics, canUpdateBankSupportedRoutingScheme, canUpdateOrganisation, canUpdateRoutingScheme, canUpdateSystemView} +import code.api.util.ApiRole.{canAttachOpenCorridorPromise, canConfigureAmqpBankBroker, canGetMessageOutbox, canRetryMessageOutbox, canSettleOpenCorridor, canCreateAccount, canCreateEntitlementAtAnyBank, canCreateEntitlementAtOneBank, canCreateMetricsArchiveRun, canCreateOrganisation, canCreateRoutingScheme, canCreateTestEmail, canCreateUtilityVendResult, canDeleteEntitlementAtAnyBank, canDeleteOrganisation, canDeleteRoutingScheme, canDeleteSchedulerJobLock, canGetAccountAccessTrace, canGetAnyOrganisation, canGetAnyUser, canGetCacheConfig, canGetCacheInfo, canGetCacheNamespaces, canGetConfig, canGetConnectorHealth, canGetCustomersAtOneBank, canGetDatabasePoolInfo, canGetMetricsDiagnostics, canGetMigrations, canGetSchedulerJobLocks, canReadMetrics, canUpdateBankSupportedRoutingScheme, canUpdateOrganisation, canUpdateRoutingScheme, canUpdateSystemView} import code.api.util.CommonsEmailWrapper import code.model.dataAccess.{AuthUser, BankAccountCreation, MappedBank, ResourceUser} import code.consent.Consents @@ -6153,6 +6153,47 @@ object Http4s700 { http4sPartialFunction = Some(getMyDynamicChangeRequests) ) + // Route: GET /obp/v7.0.0/management/dynamic-code-approval-config + // Lets a client (the API Manager create/edit pages) tell the maker up front whether a write will be + // applied or queued for approval. Authenticated, no role: any user who can create an artefact needs this. + val getDynamicCodeApprovalConfig: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "dynamic-code-approval-config" => + EndpointHelpers.withUser(req) { (_, _) => + Future.successful(JSONFactory700.DynamicCodeApprovalConfigJsonV700( + dynamic_code_execution_enabled = code.api.util.DynamicUtil.dynamicCodeExecutionEnabled, + requires_approval = MakerChecker.enabled, + target_types = if (MakerChecker.enabled) MakerChecker.managedTargetTypes.toList.sorted else Nil, + delete_requires_approval = MakerChecker.enabled && MakerChecker.requireApprovalForDelete, + request_ttl_hours = MakerChecker.requestTtlHours, + approval_role = ApiRole.canApproveDynamicChangeRequest.toString + )) + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicCodeApprovalConfig), + "GET", + "/management/dynamic-code-approval-config", + "Get Dynamic Code Approval Config", + s"""Returns whether maker/checker approval gates dynamic code and configuration on this instance, so a client can tell the maker before they submit. + | + |* `dynamic_code_execution_enabled` — the `allow_user_generated_scala_code` prop, the master kill switch. When false, creating or running any user-supplied Scala fails with $DynamicCodeExecutionDisabled, whatever the approval settings say. + |* `requires_approval` — the `dynamic_code_requires_approval` prop. When false every other field is informational and writes are applied directly. + |* `target_types` — the `dynamic_code_approval_target_types` prop: the target types whose create, update and delete calls are queued as Dynamic Change Requests. Empty when approval is off. + |* `delete_requires_approval` — the `dynamic_code_delete_requires_approval` prop, combined with `requires_approval`. + |* `request_ttl_hours` — the `dynamic_code_approval_request_ttl_hours` prop: INITIATED requests older than this expire. 0 means never. + |* `approval_role` — the Role a checker must hold to approve, reject or deactivate. + | + |$makerCheckerIntro + |No Role is required. ${userAuthenticationMessage(true)}""".stripMargin, + EmptyBody, + JSONFactory700.dynamicCodeApprovalConfigJsonV700Example, + List($AuthenticatedUserIsRequired, UnknownError), + apiTagDynamicChangeRequest :: apiTagDynamic :: apiTagApi :: Nil, + None, + http4sPartialFunction = Some(getDynamicCodeApprovalConfig) + ) + val getDynamicChangeRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> `prefixPath` / "management" / "dynamic-change-requests" / changeRequestId if changeRequestId.nonEmpty => EndpointHelpers.withUser(req) { (_, cc) => @@ -6393,6 +6434,88 @@ object Http4s700 { // // REQUIREMENT: each `val endpoint` must be declared BEFORE its `resourceDocs +=` // so that `Some(endpoint)` captures the initialized route, not null. + // ─── getConsumerRateLimits ───────────────────────────────────────────── + // Every per-consumer rate limit row on the instance, so an operator can see what overrides the + // consumer limiter's defaults without opening each consumer. Per-consumer reads stay in v5.1.0 / v6.0.0. + lazy val getConsumerRateLimits: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "rate-limits" => + EndpointHelpers.withUser(req) { (_, _) => + for { + rows <- code.ratelimiting.RateLimitingDI.rateLimiting.vend.getAll() + consumerIds = rows.map(_.consumerId).distinct + consumers <- Future.traverse(consumerIds)(id => + code.consumer.Consumers.consumers.vend.getConsumerByConsumerIdFuture(id).map(box => id -> box.map(_.name.get).openOr("")) + ) + names = consumers.toMap + now = new java.util.Date() + } yield JSONFactory700.ConsumerRateLimitsJsonV700( + rows.sortBy(r => (names.getOrElse(r.consumerId, ""), r.consumerId, r.fromDate.getTime)) + .map(r => JSONFactory700.createConsumerRateLimitJsonV700(r, names.getOrElse(r.consumerId, ""), now)) + ) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getConsumerRateLimits), + "GET", + "/management/rate-limits", + "Get Rate Limits for all Consumers", + s"""Returns every per-consumer rate limit row on this instance, across all Consumers, sorted by Consumer name. + | + |These rows are what override the consumer limiter's defaults (see Get Rate Limiter Config, whose `consumer_default` row applies + |to a Consumer with no rows of its own). `is_active` is whether `from_date`..`to_date` covers now. `api_version`, `api_name` + |and `bank_id` narrow a row to one endpoint or bank; all absent means the row applies to every call the Consumer makes. + |-1 means unlimited and 0 blocks every call. + | + |For one Consumer's rows use Get Rate Limits for a Consumer (v5.1.0); to change them use the v6.0.0 endpoints. + | + |${userAuthenticationMessage(true)} + |""".stripMargin, + EmptyBody, + JSONFactory700.consumerRateLimitsJsonV700Example, + List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), + List(apiTagRateLimits, apiTagConsumer, apiTagApi), + Some(List(ApiRole.canReadCallLimits)), + http4sPartialFunction = Some(getConsumerRateLimits) + ) + + // ─── getRateLimiterConfig ────────────────────────────────────────────── + lazy val getRateLimiterConfig: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "rate-limiter-config" => + EndpointHelpers.withUser(req) { (_, _) => + Future(JSONFactory700.createRateLimitersJsonV700()) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getRateLimiterConfig), + "GET", + "/management/rate-limiter-config", + "Get Rate Limiter Config", + s"""Returns the live configuration of the three rate limiters on this instance, in the order they are checked: + | + |1. **self_service** runs before routing and authentication, keyed by client IP address, on the endpoints anyone can call before the bank has granted them anything. A trip answers 429 `OBP-10060`. + |2. **authentication** runs inside the credential check, keyed by IP address and account. A trip answers 429 `OBP-10061`. + |3. **consumer** runs after authentication, keyed by Consumer, or by IP address for anonymous calls. A trip answers 429 `OBP-10018`. + | + |`mode` is `shadow` (trips are logged and reported in the `X-Rate-Limit-Warning` header, the request is allowed) or `enforce` (429). + |In `limits`, -1 means unlimited and 0 blocks every call; windows a limiter does not have are absent. The consumer limiter's + |`consumer_default` row is what applies to a Consumer with no rate limit rows of its own. + | + |See the Rate Limiting glossary entry. + | + |${userAuthenticationMessage(true)} + |""".stripMargin, + EmptyBody, + JSONFactory700.rateLimitersJsonV700Example, + List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), + List(apiTagRateLimits, apiTagSystem, apiTagApi), + Some(List(canGetConfig)), + http4sPartialFunction = Some(getRateLimiterConfig) + ) + val allRoutes: HttpRoutes[IO] = { val sorted = resourceDocs .sortBy(rd => -rd.requestUrl.split("/").count(_.nonEmpty)) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index e69f6be691..b2d533d2cc 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1,7 +1,7 @@ package code.api.v7_0_0 import code.api.Constant -import code.api.util.{APIUtil, CallContext, ExampleValue} +import code.api.util.{APIUtil, AuthRateLimiter, CallContext, ExampleValue, RateLimitingUtil, SelfServiceRateLimiter} import code.api.util.ErrorMessages import code.api.util.ErrorMessages.MandatoryPropertyIsNotSet import code.api.v2_0_0.EntitlementJSONs @@ -161,6 +161,92 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { case class ErrorMessageEntryJsonV700(code: String, name: String, message: String) + // ─── Rate limiter config (GET /management/rate-limiter-config) ───────────────────────── + /** One limit row of a rate limiter. Windows the limiter does not have are absent; -1 means unlimited, 0 blocks. */ + case class RateLimiterLimitJsonV700( + scope: String, + per_second: Option[Long] = None, + per_minute: Option[Long] = None, + per_hour: Option[Long] = None, + per_day: Option[Long] = None, + per_week: Option[Long] = None, + per_month: Option[Long] = None, + global_per_hour: Option[Long] = None + ) + /** One of the three rate limiters, in the order they are checked. `mode` is shadow or enforce. */ + case class RateLimiterJsonV700( + name: String, + order: Int, + error_code: String, + enabled: Boolean, + mode: String, + keyed_by: String, + runs: String, + props_prefix: String, + limits: List[RateLimiterLimitJsonV700] + ) + case class RateLimitersJsonV700(rate_limiters: List[RateLimiterJsonV700]) + + val rateLimitersJsonV700Example: RateLimitersJsonV700 = RateLimitersJsonV700(List( + RateLimiterJsonV700("self_service", 1, "OBP-10060", enabled = true, "shadow", "client IP address", + "before routing and before authentication, on the self-service endpoints", "self_service.rate_limit", + List(RateLimiterLimitJsonV700("signup", per_minute = Some(3), per_hour = Some(5), per_day = Some(10), global_per_hour = Some(500)))), + RateLimiterJsonV700("authentication", 2, "OBP-10061", enabled = false, "shadow", "client IP address and account", + "inside the credential check of Direct Login, DAuth, Gateway Login and SIWE", "auth.rate_limit", + List(RateLimiterLimitJsonV700("ip", per_minute = Some(10), per_hour = Some(100)), RateLimiterLimitJsonV700("account", per_minute = Some(6)))), + RateLimiterJsonV700("consumer", 3, "OBP-10018", enabled = true, "enforce", "Consumer, or client IP address for anonymous calls", + "after authentication, on every endpoint", "rate_limiting_per_*", + List(RateLimiterLimitJsonV700("consumer_default", Some(-1), Some(-1), Some(-1), Some(-1), Some(-1), Some(-1)), RateLimiterLimitJsonV700("anonymous", per_hour = Some(1000)))) + )) + + /** The live configuration of the three rate limiters, from props and built-in defaults. */ + def createRateLimitersJsonV700(): RateLimitersJsonV700 = { + def errorCode(msg: String): String = APIUtil.extractErrorMessageCode(msg) + def prop(name: String, default: Long): Long = APIUtil.getPropsAsLongValue(name, default) + def opt(v: Long): Option[Long] = Some(v) + + val selfService = RateLimiterJsonV700( + name = "self_service", order = 1, error_code = errorCode(ErrorMessages.TooManyRequestsSelfService), + enabled = SelfServiceRateLimiter.enabled, mode = SelfServiceRateLimiter.mode, + keyed_by = "client IP address", + runs = "before routing and before authentication, on the self-service endpoints", + props_prefix = SelfServiceRateLimiter.PropsPrefix, + limits = SelfServiceRateLimiter.scopeDefaults.keys.toList.sorted.map { scope => + RateLimiterLimitJsonV700(scope, + per_minute = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_minute")), + per_hour = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_hour")), + per_day = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_day")), + global_per_hour = opt(SelfServiceRateLimiter.globalPerHourLimit(scope))) + } + ) + val authentication = RateLimiterJsonV700( + name = "authentication", order = 2, error_code = errorCode(ErrorMessages.TooManyRequestsAuth), + enabled = AuthRateLimiter.enabled, mode = AuthRateLimiter.mode, + keyed_by = "client IP address and account", + runs = "inside the credential check of Direct Login, DAuth, Gateway Login and SIWE", + props_prefix = AuthRateLimiter.PropsPrefix, + limits = List( + RateLimiterLimitJsonV700("ip", per_minute = opt(AuthRateLimiter.perIpPerMinute), per_hour = opt(AuthRateLimiter.perIpPerHour)), + RateLimiterLimitJsonV700("account", per_minute = opt(AuthRateLimiter.perUserPerMinute))) + ) + val consumer = RateLimiterJsonV700( + name = "consumer", order = 3, error_code = errorCode(ErrorMessages.TooManyRequests), + enabled = RateLimitingUtil.useConsumerLimits, mode = "enforce", + keyed_by = "Consumer, or client IP address for anonymous calls", + runs = "after authentication, on every endpoint", + props_prefix = "rate_limiting_per_*", + limits = List( + // Props defaults apply to a Consumer with no rate limit rows; rows written by the management + // endpoints and API Product Subscriptions override them per Consumer. + RateLimiterLimitJsonV700("consumer_default", + per_second = opt(prop("rate_limiting_per_second", -1)), per_minute = opt(prop("rate_limiting_per_minute", -1)), + per_hour = opt(prop("rate_limiting_per_hour", -1)), per_day = opt(prop("rate_limiting_per_day", -1)), + per_week = opt(prop("rate_limiting_per_week", -1)), per_month = opt(prop("rate_limiting_per_month", -1))), + RateLimiterLimitJsonV700("anonymous", per_hour = opt(prop("user_consumer_limit_anonymous_access", 1000)))) + ) + RateLimitersJsonV700(List(selfService, authentication, consumer)) + } + // Cached for server lifetime: ErrorMessages is a static catalog of `val X = "OBP-NNNNN: ..."` // strings, so reflecting over it once at first access is sufficient. Filters: // - only String-typed fields (skips synthetic lazy-val bitmaps and helper defs) @@ -1565,6 +1651,90 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { sca_enabled = true ) + // ─── Consumer rate limits across all consumers — what overrides the consumer limiter's defaults ── + + case class ConsumerRateLimitJsonV700( + rate_limiting_id: String, + consumer_id: String, + consumer_name: String, + api_version: Option[String], + api_name: Option[String], + bank_id: Option[String], + from_date: java.util.Date, + to_date: java.util.Date, + is_active: Boolean, + per_second_call_limit: String, + per_minute_call_limit: String, + per_hour_call_limit: String, + per_day_call_limit: String, + per_week_call_limit: String, + per_month_call_limit: String, + created_at: java.util.Date, + updated_at: java.util.Date + ) + case class ConsumerRateLimitsJsonV700(rate_limits: List[ConsumerRateLimitJsonV700]) + + def createConsumerRateLimitJsonV700(r: code.ratelimiting.RateLimiting, consumerName: String, now: java.util.Date): ConsumerRateLimitJsonV700 = + ConsumerRateLimitJsonV700( + rate_limiting_id = r.rateLimitingId, + consumer_id = r.consumerId, + consumer_name = consumerName, + api_version = r.apiVersion, + api_name = r.apiName, + bank_id = r.bankId, + from_date = r.fromDate, + to_date = r.toDate, + is_active = !now.before(r.fromDate) && !now.after(r.toDate), + per_second_call_limit = r.perSecondCallLimit.toString, + per_minute_call_limit = r.perMinuteCallLimit.toString, + per_hour_call_limit = r.perHourCallLimit.toString, + per_day_call_limit = r.perDayCallLimit.toString, + per_week_call_limit = r.perWeekCallLimit.toString, + per_month_call_limit = r.perMonthCallLimit.toString, + created_at = r.createdAt.get, + updated_at = r.updatedAt.get + ) + + lazy val consumerRateLimitsJsonV700Example = ConsumerRateLimitsJsonV700(List(ConsumerRateLimitJsonV700( + rate_limiting_id = "2f1b6c0e-9d5a-4c3b-8e7f-1a2b3c4d5e6f", + consumer_id = "8e716299-4668-4efd-976a-67f57a9984ec", + consumer_name = "Mobile App", + api_version = None, + api_name = None, + bank_id = None, + from_date = APIUtil.DateWithDayExampleObject, + to_date = APIUtil.DateWithDayExampleObject, + is_active = true, + per_second_call_limit = "-1", + per_minute_call_limit = "-1", + per_hour_call_limit = "1000", + per_day_call_limit = "10000", + per_week_call_limit = "-1", + per_month_call_limit = "-1", + created_at = APIUtil.DateWithDayExampleObject, + updated_at = APIUtil.DateWithDayExampleObject + ))) + + // ─── Dynamic code approval config — whether maker/checker gates dynamic artefacts on this instance ── + + case class DynamicCodeApprovalConfigJsonV700( + dynamic_code_execution_enabled: Boolean, + requires_approval: Boolean, + target_types: List[String], + delete_requires_approval: Boolean, + request_ttl_hours: Int, + approval_role: String + ) + + lazy val dynamicCodeApprovalConfigJsonV700Example = DynamicCodeApprovalConfigJsonV700( + dynamic_code_execution_enabled = true, + requires_approval = true, + target_types = List("DYNAMIC_RESOURCE_DOC", "DYNAMIC_MESSAGE_DOC", "CONNECTOR_METHOD", "ABAC_RULE"), + delete_requires_approval = true, + request_ttl_hours = 168, + approval_role = "CanApproveDynamicChangeRequest" + ) + // ─── User JSON — v7 adds the user's own OBP-verified mobile phone fields ─────── // Distinct from Customer.mobile_phone_number (bank-scoped KYC data): this is the // authenticated person's number, global across banks, stored on ResourceUser. diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala index 82f0c05fc8..8dd4c43f89 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala @@ -2,6 +2,7 @@ package code.obp.grpc.signal import code.api.cache.RedisMessaging import code.api.util.ErrorMessages.{InvalidJsonFormat, InvalidSignalChannelName, SignalMessageContainsDangerousCharacters, SignalMessageTooLong} +import code.api.util.SelfServiceRateLimiter import code.api.v6_0_0.{PostSignalMessageJsonV600, SignalMessageJsonV600} import code.obp.grpc.chat.AuthInterceptor import code.obp.grpc.signal.api._ @@ -87,6 +88,19 @@ object SignalChannelsServiceImpl extends SignalChannelsServiceGrpc.SignalChannel message_type = Option(request.messageType).filter(_.nonEmpty), to_user_id = Option(request.toUserId).filter(_.nonEmpty)) val consumerId = callContext.flatMap(_.consumer match { case Full(c) => Some(c.consumerId.get); case _ => None }).getOrElse("") + // Channel creation cap (scope signal_channel_create). REST keys this on the client IP in + // SelfServiceRateLimitMiddleware; gRPC has no request IP in scope, so it keys on the + // Consumer (falling back to the User). Publishing to an existing channel is never counted. + if (RedisMessaging.channelInfo(request.channelName).isEmpty) { + val key = if (consumerId.nonEmpty) consumerId else user.userId + SelfServiceRateLimiter.check("signal_channel_create", key, "consumer") match { + case SelfServiceRateLimiter.Blocked(scope, _, exceeded) => + throw Status.RESOURCE_EXHAUSTED + .withDescription(SelfServiceRateLimiter.blockedMessage(scope, exceeded)) + .asRuntimeException() + case _ => // Allowed, Warned (already logged) or Skipped + } + } val published = SignalChannels.publish(request.channelName, user.userId, consumerId, post) PublishResponse( messageId = published.message_id, diff --git a/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala new file mode 100644 index 0000000000..1141424d20 --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala @@ -0,0 +1,191 @@ +package code.api.util + +import cats.effect.IO +import code.api.util.SelfServiceRateLimiter._ +import code.api.util.http4s.SelfServiceRateLimitMiddleware +import code.setup.ServerSetup +import org.http4s.{Method, Request, Uri} + +import java.util.concurrent.atomic.AtomicLong + +class SelfServiceRateLimiterTest extends ServerSetup { + + // Unique keys per scenario so Redis counters from different scenarios never collide. + private val counter = new AtomicLong(System.nanoTime()) + private def freshIp(): String = s"198.51.100.${counter.incrementAndGet() % 255 + 1}.${counter.get()}" + private def freshScope(): String = s"testscope_${counter.incrementAndGet()}" + + private val P = SelfServiceRateLimiter.PropsPrefix + + feature("SelfServiceRateLimiter") { + + scenario("disabled: returns Skipped and counts nothing") { + setPropsValues(s"$P.enabled" -> "false") + SelfServiceRateLimiter.check("signup", freshIp()) shouldBe Skipped("signup") + } + + scenario("default mode is shadow: a trip returns Warned, never Blocked") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "", + s"$P.$scope.per_ip.per_minute" -> "1", + s"$P.$scope.per_ip.per_hour" -> "1000", + s"$P.$scope.per_ip.per_day" -> "1000" + ) + val ip = freshIp() + SelfServiceRateLimiter.check(scope, ip) shouldBe a[Allowed] + val second = SelfServiceRateLimiter.check(scope, ip) + second shouldBe a[Warned] + val Warned(_, _, exceeded) = second + exceeded.name shouldBe "ip_per_minute" + exceeded.limit shouldBe 1L + exceeded.current shouldBe 2L + } + + scenario("the warning text carries OBP-10059, the scope and the limit; no date unless announced") { + val scope = freshScope() + val exceeded = Window("ip_per_hour", RateLimitingPeriod.PER_HOUR, limit = 5, current = 6, resetSeconds = 100) + setPropsValues(s"$P.enforce_announced_from" -> "") + val plain = SelfServiceRateLimiter.warningMessage(scope, exceeded) + plain shouldBe s"OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate limit for $scope (5 per hour) in the future." + + setPropsValues(s"$P.enforce_announced_from" -> "2026-10-01") + SelfServiceRateLimiter.warningMessage(scope, exceeded) should endWith(" in the future, from 2026-10-01.") + } + + scenario("enforce mode: the (limit+1)th request is Blocked, with reset within the window") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "enforce", + s"$P.$scope.per_ip.per_minute" -> "2", + s"$P.$scope.per_ip.per_hour" -> "1000", + s"$P.$scope.per_ip.per_day" -> "1000" + ) + val ip = freshIp() + SelfServiceRateLimiter.check(scope, ip) shouldBe a[Allowed] + SelfServiceRateLimiter.check(scope, ip) shouldBe a[Allowed] + val third = SelfServiceRateLimiter.check(scope, ip) + third shouldBe a[Blocked] + val Blocked(_, _, exceeded) = third + exceeded.name shouldBe "ip_per_minute" + exceeded.resetSeconds should (be > 0L and be <= 60L) + SelfServiceRateLimiter.blockedMessage(scope, exceeded) should startWith(ErrorMessages.TooManyRequestsSelfService) + SelfServiceRateLimiter.blockedMessage(scope, exceeded) should include("OBP-10060") + } + + scenario("different IPs do not share per-IP counters") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "enforce", + s"$P.$scope.per_ip.per_minute" -> "1", + s"$P.$scope.per_ip.per_hour" -> "1000", + s"$P.$scope.per_ip.per_day" -> "1000" + ) + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe a[Allowed] + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe a[Allowed] + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe a[Allowed] + } + + scenario("a window set to -1 is switched off; -1 everywhere means Skipped") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "enforce", + s"$P.$scope.per_ip.per_minute" -> "-1", + s"$P.$scope.per_ip.per_hour" -> "-1", + s"$P.$scope.per_ip.per_day" -> "-1", + s"$P.$scope.global.per_hour" -> "-1" + ) + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe Skipped(scope) + } + + scenario("the global per-hour cap trips across different IPs") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "enforce", + s"$P.$scope.per_ip.per_minute" -> "1000", + s"$P.$scope.per_ip.per_hour" -> "1000", + s"$P.$scope.per_ip.per_day" -> "1000", + s"$P.$scope.global.per_hour" -> "2" + ) + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe a[Allowed] + SelfServiceRateLimiter.check(scope, freshIp()) shouldBe a[Allowed] + val third = SelfServiceRateLimiter.check(scope, freshIp()) + third shouldBe a[Blocked] + val Blocked(_, _, exceeded) = third + exceeded.name shouldBe "global_per_hour" + } + + scenario("an unknown IP disables the per-IP windows but keeps the global one") { + val scope = freshScope() + setPropsValues( + s"$P.enabled" -> "true", + s"$P.mode" -> "enforce", + s"$P.$scope.per_ip.per_minute" -> "1", + s"$P.$scope.global.per_hour" -> "1000" + ) + val outcome = SelfServiceRateLimiter.check(scope, "Unknown") + outcome shouldBe a[Allowed] + outcome.windows.map(_.name) shouldBe List("global_per_hour") + } + + scenario("limit resolution: scope prop beats generic prop beats built-in defaults") { + val scope = freshScope() + setPropsValues( + s"$P.per_ip.per_minute" -> "7", + s"$P.$scope.per_ip.per_minute" -> "3", + s"$P.$scope.per_ip.per_hour" -> "", + s"$P.per_ip.per_hour" -> "", + s"$P.signup.per_ip.per_hour" -> "" + ) + SelfServiceRateLimiter.perKeyLimit(scope, "per_minute") shouldBe 3L + SelfServiceRateLimiter.perKeyLimit(scope, "per_hour") shouldBe genericDefaults.perHour + SelfServiceRateLimiter.perKeyLimit("signup", "per_hour") shouldBe scopeDefaults("signup").perHour + SelfServiceRateLimiter.globalPerHourLimit("signup") shouldBe scopeDefaults("signup").globalPerHour + } + + scenario("tightest window is the one with the fewest remaining calls") { + val windows = List( + Window("ip_per_minute", RateLimitingPeriod.PER_MINUTE, limit = 10, current = 1, resetSeconds = 50), + Window("ip_per_hour", RateLimitingPeriod.PER_HOUR, limit = 60, current = 58, resetSeconds = 900), + Window("ip_per_day", RateLimitingPeriod.PER_DAY, limit = 200, current = 58, resetSeconds = 80000) + ) + Allowed("x", windows).tightest.map(_.name) shouldBe Some("ip_per_hour") + } + } + + feature("SelfServiceRateLimitMiddleware scope table") { + + def post(path: String): Request[IO] = Request[IO](Method.POST, Uri.unsafeFromString(path)) + def get(path: String): Request[IO] = Request[IO](Method.GET, Uri.unsafeFromString(path)) + + scenario("self-service paths map to their scopes, for any API version") { + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/users")) shouldBe Some("signup") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v7.0.0/users")) shouldBe Some("signup") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/users/email-validation")) shouldBe Some("signup") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v4.0.0/banks/gh.29.uk/user-invitations")) shouldBe Some("signup") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/users/password-reset-url")) shouldBe Some("password_reset") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/users/password")) shouldBe Some("password_reset") + // logins belong to AuthRateLimiter, not to the self-service table + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/my/logins/direct")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(post("/my/logins/direct")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(post("/my/logins/siwe/challenge")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v5.0.0/consumer/consent-requests")) shouldBe Some("consent_request") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/consumer/vrp-consent-requests")) shouldBe Some("consent_request") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/dynamic-registration/consumers")) shouldBe Some("consumer_registration") + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v4.0.0/account/check/scheme/iban")) shouldBe Some("lookup") + } + + scenario("everything else is left alone") { + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/users")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/users/current")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/users/USER_ID/attributes")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v6.0.0/banks")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/signal-channels")) shouldBe None + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala new file mode 100644 index 0000000000..8a1a0c1506 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala @@ -0,0 +1,66 @@ +package code.api.v7_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole +import code.api.util.ErrorMessages._ +import code.api.v7_0_0.Http4s700.Implementations7_0_0 +import code.entitlement.Entitlement +import code.setup.ServerSetupWithTestData +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST.{JArray, JObject} +import org.scalatest.Tag + +/** GET /obp/v7.0.0/management/rate-limiter-config: the three limiters, in check order, each with its 429 code. */ +class RateLimitersTest extends ServerSetupWithTestData { + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint1 extends Tag(nameOf(Implementations7_0_0.getRateLimiterConfig)) + + private def v7 = baseRequest / "obp" / "v7.0.0" + private def str(json: org.json4s.JValue, field: String): String = (json \ field).values.toString + + feature("Get Rate Limiters") { + scenario("unauthenticated is 401", ApiEndpoint1, VersionOfApi) { + val response = makeGetRequest(v7 / "management" / "rate-limiter-config") + response.code should equal(401) + response.body.toString should include(AuthenticatedUserIsRequired.split(":").head) + } + + scenario("without CanGetConfig is 403", ApiEndpoint1, VersionOfApi) { + val response = makeGetRequest((v7 / "management" / "rate-limiter-config").GET <@ (user1)) + response.code should equal(403) + response.body.toString should include(UserHasMissingRoles) + response.body.toString should include(ApiRole.canGetConfig.toString) + } + + scenario("with CanGetConfig the three limiters come back in check order with distinct 429 codes", ApiEndpoint1, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetConfig.toString) + val response = makeGetRequest((v7 / "management" / "rate-limiter-config").GET <@ (user1)) + response.code should equal(200) + + val limiters = (response.body \ "rate_limiters").asInstanceOf[JArray].arr + limiters.map(l => str(l, "name")) should equal(List("self_service", "authentication", "consumer")) + limiters.map(l => str(l, "order")) should equal(List("1", "2", "3")) + limiters.map(l => str(l, "error_code")) should equal(List("OBP-10060", "OBP-10061", "OBP-10018")) + limiters.foreach { l => + List("shadow", "enforce") should contain(str(l, "mode")) + (l \ "limits").asInstanceOf[JArray].arr should not be empty + } + + Then("the self-service limiter lists its scopes but not login, which belongs to the authentication limiter") + val selfServiceScopes = (limiters.head \ "limits").asInstanceOf[JArray].arr.map(l => str(l, "scope")) + selfServiceScopes should contain allOf ("signup", "password_reset", "consumer_registration") + selfServiceScopes should not contain "login" + + Then("the authentication limiter reports an ip and an account window") + val authScopes = (limiters(1) \ "limits").asInstanceOf[JArray].arr.map(l => str(l, "scope")) + authScopes should equal(List("ip", "account")) + + Then("the consumer limiter reports the props defaults and the anonymous ceiling") + val consumerRows = (limiters(2) \ "limits").asInstanceOf[JArray].arr + consumerRows.map(l => str(l, "scope")) should equal(List("consumer_default", "anonymous")) + (consumerRows.last \ "per_hour").values.toString.toLong should be > 0L + } + } +} From bdcb4e671199522994e8faf46ad721c0e18925f8 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 6 Sep 2026 22:57:09 +0200 Subject: [PATCH 06/13] adding POST /obp/v7.0.0/management/dynamic-resource-docs/compile --- .../endpoint/helper/DynamicEndpoints.scala | 134 ++++++++++++------ .../scala/code/api/util/DynamicUtil.scala | 32 +++++ .../main/scala/code/api/util/Glossary.scala | 19 +++ .../scala/code/api/v7_0_0/Http4s700.scala | 78 ++++++++++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 31 ++++ 5 files changed, 251 insertions(+), 43 deletions(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index beaad62688..a5d073dbac 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -77,6 +77,88 @@ trait EndpointGroup { * @param successResponseBody successResponseBody from the post json body,it is JValue here. * @param methodBody it is url-encoded string for the api level code. */ +object CompiledObjects { + /** + * The native http4s template a method body is inlined into. Returns the full source and the + * 1-based line on which the method body starts, so compiler positions can be mapped back to it. + * The compiled artifact is an `Http4sEndpointIO` (PartialFunction[Request[IO], CallContext => IO[Response[IO]]]). + * `DynamicCompileEndpoint._` injects the `OBPReturnType[T] => IO[Response[IO]]` implicit (so the + * familiar `Future.successful((json, HttpCode.`200`(cc)))` body style works) and the + * `errorResponse(msg, code)` helper (replacing `Full(errorJsonResponse(...))`). + */ + def wrapMethodBody(requestBodyCaseClasses: String, responseBodyCaseClasses: String, decodedMethodBody: String): (String, Int) = { + val prefix = + s""" + |import cats.effect.IO + |import org.http4s.{Request, Response} + |import code.api.util.CallContext + |import code.api.util.ErrorMessages.{InvalidJsonFormat, InvalidRequestPayload} + |import code.api.util.NewStyle.HttpCode + |import code.api.util.APIUtil.OBPReturnType + |import org.json4s.MappingException + |import code.api.dynamic.endpoint.helper.DynamicCompileEndpoint._ + | + |import scala.concurrent.Future + |import com.openbankproject.commons.ExecutionContext.Implicits.global + |import net.liftweb.common.{Box, Empty, Failure, Full} + | + |implicit val formats = code.api.util.CustomJsonFormats.formats + | + |$requestBodyCaseClasses + | + |$responseBodyCaseClasses + | + |val endpoint: code.api.util.APIUtil.Http4sEndpointIO = { + | case request => { callContext => + | val Some(pathParams) = callContext.resourceDocument.map(_.getPathParams(request.uri.path.segments.toList.map(_.encoded))) + | """.stripMargin + val suffix = + s""" + | } + |} + | + |endpoint + | + |""".stripMargin + val bodyStartLine = prefix.count(_ == '\n') + 1 + (prefix + decodedMethodBody + suffix, bodyStartLine) + } + + /** + * Dry run without constructing a CompiledObjects (whose constructor compiles for real): compiler + * diagnostics with line numbers relative to the method body the author wrote. Empty = compiles. + */ + def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String): List[DynamicUtil.CompileProblem] = { + val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") + val requestBody: Product = exampleRequestBody match { + case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None) + case _ => toCaseObject(exampleRequestBody) + } + val successResponse: Product = toCaseObject(successResponseBody) + val requestExample: Option[JValue] = if (requestBody.isInstanceOf[PrimaryDataBody[_]]) None else exampleRequestBody + val responseExample: Option[JValue] = if (successResponse.isInstanceOf[PrimaryDataBody[_]]) None else successResponseBody + val (requestBodyCaseClasses, responseBodyCaseClasses) = DynamicEndpointCodeGenerator.buildCaseClasses(requestExample, responseExample) + val (code, bodyStartLine) = wrapMethodBody(requestBodyCaseClasses, responseBodyCaseClasses, decodedMethodBody) + DynamicUtil.checkScalaCode(code).map { p => + if (p.line > 0) p.copy(line = p.line - bodyStartLine + 1) else p + } + } + + def toCaseObject(jValue: Option[JValue]): Product = { + if (jValue.isEmpty || jValue.exists(JNothing == _)) { + EmptyBody + } else { + jValue.orNull match { + case JBool(b) => BooleanBody(b) + case JInt(l) => LongBody(l.toLong) + case JDouble(d) => DoubleBody(d) + case JString(s) => StringBody(s) + case v => DynamicUtil.toCaseObject(v) + } + } + } +} + case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { @@ -113,36 +195,7 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo // `DynamicCompileEndpoint._` injects the `OBPReturnType[T] => IO[Response[IO]]` implicit (so the // familiar `Future.successful((json, HttpCode.`200`(cc)))` body style still works) and the // `errorResponse(msg, code)` helper (replacing `Full(errorJsonResponse(...))`). - val code = - s""" - |import cats.effect.IO - |import org.http4s.{Request, Response} - |import code.api.util.CallContext - |import code.api.util.ErrorMessages.{InvalidJsonFormat, InvalidRequestPayload} - |import code.api.util.NewStyle.HttpCode - |import code.api.util.APIUtil.OBPReturnType - |import org.json4s.MappingException - |import code.api.dynamic.endpoint.helper.DynamicCompileEndpoint._ - | - |import scala.concurrent.Future - |import com.openbankproject.commons.ExecutionContext.Implicits.global - | - |implicit val formats = code.api.util.CustomJsonFormats.formats - | - |$requestBodyCaseClasses - | - |$responseBodyCaseClasses - | - |val endpoint: code.api.util.APIUtil.Http4sEndpointIO = { - | case request => { callContext => - | val Some(pathParams) = callContext.resourceDocument.map(_.getPathParams(request.uri.path.segments.toList.map(_.encoded))) - | $decodedMethodBody - | } - |} - | - |endpoint - | - |""".stripMargin + val (code, _) = CompiledObjects.wrapMethodBody(requestBodyCaseClasses, responseBodyCaseClasses, decodedMethodBody) val endpointMethod = DynamicUtil.compileScalaCode[Http4sEndpointIO](code) endpointMethod match { @@ -160,6 +213,13 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo */ def validateDependency() = Validation.validateDependency(this.partialFunction) + /** + * Dry run: compiler diagnostics for this body, with line numbers relative to the method body the + * author wrote (the wrapper's own lines are subtracted). Empty = compiles. Nothing is evaluated or cached. + */ + def compileProblems(): List[DynamicUtil.CompileProblem] = + CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody) + /** * This is used to check the security permission at the run time. * all the obp partialFunctions will be wrapped into the sandbox which under the permission control. @@ -184,17 +244,5 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } } - private def toCaseObject(jValue: Option[JValue]): Product = { - if (jValue.isEmpty || jValue.exists(JNothing == _)) { - EmptyBody - } else { - jValue.orNull match { - case JBool(b) => BooleanBody(b) - case JInt(l) => LongBody(l.toLong) - case JDouble(d) => DoubleBody(d) - case JString(s) => StringBody(s) - case v => DynamicUtil.toCaseObject(v) - } - } - } + private def toCaseObject(jValue: Option[JValue]): Product = CompiledObjects.toCaseObject(jValue) } diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 862bf609c8..a70ceccf17 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -43,6 +43,38 @@ object DynamicUtil extends MdcLoggable{ } val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() + + /** One compiler diagnostic from [[checkScalaCode]]. `line`/`column` are 1-based in the code that was checked; 0 when unknown. */ + case class CompileProblem(line: Int, column: Int, severity: String, message: String) + + /** Collects compiler diagnostics with positions; the default toolbox front end only keeps the messages. */ + private class CollectingFrontEnd extends scala.tools.reflect.FrontEnd { + // FrontEnd.log already records every diagnostic in `infos`; nothing to print. + override def display(info: Info): Unit = () + } + private val checkFrontEnd = new CollectingFrontEnd + // A second toolbox, so a dry-run check never touches the memoised compile results of the real one. + private val checkToolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox(frontEnd = checkFrontEnd) + + /** + * Compile `code` for diagnostics only: nothing is evaluated, nothing is cached. Empty list = compiles. + * Toolboxes are not thread-safe, so checks are serialised. Callers must apply the kill switch and a role. + */ + def checkScalaCode(code: String): List[CompileProblem] = checkToolBox.synchronized { + checkFrontEnd.reset() + val failure: Option[String] = try { + checkToolBox.typecheck(checkToolBox.parse(code)) + None + } catch { + case e: ToolBoxError => Some(e.message) + } + val collected = checkFrontEnd.infos.toList.filter(_.severity == checkFrontEnd.ERROR).map { info => + val (line, column) = if (info.pos != null && info.pos.isDefined) (info.pos.line, info.pos.column) else (0, 0) + CompileProblem(line, column, "ERROR", info.msg) + } + if (collected.nonEmpty) collected + else failure.map(m => CompileProblem(0, 0, "ERROR", m.stripPrefix("reflective typecheck has failed:").stripPrefix("reflective compilation has failed:").trim)).toList + } private val memoClassPool = new Memo[ClassLoader, ClassPool] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index fcf90bfe99..1dcd6d8554 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -3761,6 +3761,25 @@ object Glossary extends MdcLoggable { | |A helper endpoint (`POST /management/dynamic-resource-docs/endpoint-code`) can generate a method-body template from example request / response bodies. | +|**The method body** +| +|The body is inlined, unchanged, into a native http4s handler. Do not wrap it in a method or class. In scope: +| +|* `callContext: CallContext` - the authenticated User, Consumer, `resourceDocument`, `httpBody` (the raw request body as a String) and `callContext.callContext` (the same as an `Option`). +|* `request: org.http4s.Request[IO]` - the raw request, for headers or the URI. +|* `pathParams: Map[String, String]` - one entry per UPPER_CASE segment of the request URL, for example `pathParams("BANK_ID")`. +|* `RequestRootJsonClass` / `ResponseRootJsonClass` - case classes generated from the example request body and success response body. Parse the request with `JsonAliases.parse(rawBody).extract[RequestRootJsonClass]`. +|* `errorResponse(message, code = 400)` - returns the standard OBP error JSON with that status. +|* Imports: `Future` and the OBP execution context, `HttpCode`, `OBPReturnType`, `ErrorMessages.{InvalidJsonFormat, InvalidRequestPayload}`, `MappingException`, `cats.effect.IO`, `net.liftweb.common.{Box, Empty, Failure, Full}` (for matching on Connector results), and an implicit `formats`. +| +|The last expression is the response. Return `Future.successful((value, HttpCode.`200`(callContext)))` - any case class, Map or List that serialises to JSON, paired with the call context whose status was set by `HttpCode` - or `errorResponse(...)`. An implicit converts that `Future[(T, Option[CallContext])]` into the http4s response. The Lift-era shapes `Full(successJsonResponse(...))` and `Box[JsonResponse]` are not accepted; a body that returns them fails to compile (`OBP-40045`). +| +|The smallest valid body: +| +| Future.successful((Map("hello" -> "world"), HttpCode.`200`(callContext))) +| +|To check a body before creating anything, `POST /obp/v7.0.0/management/dynamic-resource-docs/compile` compiles it the same way and returns the compiler's errors with line numbers relative to the body. The API Manager's Create page uses it for its Compile button and for the loop in which Opey rewrites the body until it compiles. +| |See ${getGlossaryItemLink("Dynamic Code Paths")} for how Dynamic Resource Docs relate to the other runtime-defined building blocks, and ${getGlossaryItemLink("Dynamic Change Request")} for how an operator can require a second person to approve each definition before it is compiled and served. | """.stripMargin) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 8d678f16b1..215f6e92f2 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -6153,6 +6153,84 @@ object Http4s700 { http4sPartialFunction = Some(getMyDynamicChangeRequests) ) + // Route: POST /obp/v7.0.0/management/dynamic-resource-docs/compile + // Dry run for tooling: compile a method body and report the compiler's diagnostics, mapped to the + // body's own line numbers. Nothing is stored, evaluated or cached. Compiling is still a full scalac run + // that can execute code at top level, so it needs the create role, the kill switch, and a per-user throttle. + private val dynamicCompileCallsPerMinute = 20 + private val dynamicCompileCalls = new java.util.concurrent.ConcurrentHashMap[String, java.util.ArrayDeque[Long]]() + private def allowDynamicCompile(userId: String): Boolean = dynamicCompileCalls.synchronized { + val now = System.currentTimeMillis() + val q = dynamicCompileCalls.computeIfAbsent(userId, _ => new java.util.ArrayDeque[Long]()) + while (!q.isEmpty && q.peekFirst() < now - 60000L) q.pollFirst() + if (q.size() >= dynamicCompileCallsPerMinute) false else { q.addLast(now); true } + } + + val compileDynamicResourceDoc: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "dynamic-resource-docs" / "compile" => + EndpointHelpers.withUser(req) { (u, cc) => + import code.api.v7_0_0.JSONFactory700.{DynamicCompileErrorJsonV700, DynamicCompileResultJsonV700, DynamicResourceDocCompileJsonV700} + for { + _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { code.api.util.DynamicUtil.dynamicCodeExecutionEnabled } + _ <- code.util.Helper.booleanToFuture(s"${code.api.util.ErrorMessages.TooManyRequests} at most $dynamicCompileCallsPerMinute dry-run compiles per minute per user", 429, Some(cc)) { allowDynamicCompile(u.userId) } + body <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the ${classOf[DynamicResourceDocCompileJsonV700].getSimpleName}", 400, Some(cc)) { + com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[DynamicResourceDocCompileJsonV700] + } + _ <- code.util.Helper.booleanToFuture(s"""$InvalidJsonFormat The request_verb must be one of ["POST", "PUT", "GET", "DELETE"]""", cc = Some(cc)) { + Set("POST", "PUT", "GET", "DELETE").contains(body.request_verb) + } + result <- Future { + val start = System.currentTimeMillis() + val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body)) match { + case scala.util.Success(ps) => ps + case scala.util.Failure(e) => List(code.api.util.DynamicUtil.CompileProblem(0, 0, "ERROR", Option(e.getMessage).getOrElse(e.toString))) + } + val dependencyError: Option[String] = + if (problems.nonEmpty) None + else scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects(body.example_request_body, body.success_response_body, body.method_body).validateDependency()) match { + case scala.util.Success(_) => None + case scala.util.Failure(e: code.api.JsonResponseException) => Some(com.openbankproject.commons.util.JsonAliases.compactRender(e.jsonResponse.body)) + case scala.util.Failure(e) => Some(Option(e.getMessage).getOrElse(e.toString)) + } + DynamicCompileResultJsonV700( + compiles = problems.isEmpty && dependencyError.isEmpty, + errors = problems.map(p => DynamicCompileErrorJsonV700(p.line, p.column, p.severity, p.message)), + dependency_error = dependencyError, + duration_ms = System.currentTimeMillis() - start + ) + } + } yield result + } + } + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(compileDynamicResourceDoc), + "POST", + "/management/dynamic-resource-docs/compile", + "Compile Dynamic Resource Doc (dry run)", + s"""Compiles a Dynamic Resource Doc method body exactly as Create Dynamic Resource Doc would, and reports the result without storing anything. + | + |Send the fields that shape the compiled code: `request_verb`, `request_url`, the URL-encoded `method_body`, and the optional + |`example_request_body` and `success_response_body` (they become the generated `RequestRootJsonClass` / `ResponseRootJsonClass`). + | + |`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's wrapper lines are + |subtracted; 0 when the compiler gave no position). When the body compiles and `dynamic_code_compile_validate_enable` is on, + |the dependency validator runs too and any forbidden call is reported in `dependency_error`. `compiles` is true only when both pass. + | + |Nothing is evaluated or cached, but compiling is a full scalac run, so the same rules apply as for creating: the + |`allow_user_generated_scala_code` kill switch, the create role, and at most $dynamicCompileCallsPerMinute calls per minute per user. + | + |Built for editors that let an author, or an assistant such as Opey, iterate on a body until it compiles before submitting it. + | + |${userAuthenticationMessage(true)}""".stripMargin, + JSONFactory700.dynamicResourceDocCompileJsonV700Example, + JSONFactory700.dynamicCompileResultJsonV700Example, + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.TooManyRequests, UnknownError), + apiTagDynamicResourceDoc :: apiTagDynamic :: Nil, + Some(List(ApiRole.canCreateDynamicResourceDoc)), + http4sPartialFunction = Some(compileDynamicResourceDoc) + ) + // Route: GET /obp/v7.0.0/management/dynamic-code-approval-config // Lets a client (the API Manager create/edit pages) tell the maker up front whether a write will be // applied or queued for approval. Authenticated, no role: any user who can create an artefact needs this. diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index b2d533d2cc..eecf017b40 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1715,6 +1715,37 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { updated_at = APIUtil.DateWithDayExampleObject ))) + // ─── Dynamic resource doc dry-run compile ───────────────────────────────── + + /** Request: the parts of a Dynamic Resource Doc that shape the compiled code. */ + case class DynamicResourceDocCompileJsonV700( + request_verb: String, + request_url: String, + method_body: String, + example_request_body: Option[JValue], + success_response_body: Option[JValue] + ) + case class DynamicCompileErrorJsonV700(line: Int, column: Int, severity: String, message: String) + case class DynamicCompileResultJsonV700( + compiles: Boolean, + errors: List[DynamicCompileErrorJsonV700], + dependency_error: Option[String], + duration_ms: Long + ) + lazy val dynamicResourceDocCompileJsonV700Example = DynamicResourceDocCompileJsonV700( + request_verb = "GET", + request_url = "/hello/world", + method_body = java.net.URLEncoder.encode("Future.successful((Map(\"hello\" -> \"world\"), HttpCode.`200`(callContext)))", "UTF-8"), + example_request_body = None, + success_response_body = Some(org.json4s.JsonAST.JObject(List(org.json4s.JsonAST.JField("hello", org.json4s.JsonAST.JString("world"))))) + ) + lazy val dynamicCompileResultJsonV700Example = DynamicCompileResultJsonV700( + compiles = false, + errors = List(DynamicCompileErrorJsonV700(1, 24, "ERROR", "not found: value Full")), + dependency_error = None, + duration_ms = 850 + ) + // ─── Dynamic code approval config — whether maker/checker gates dynamic artefacts on this instance ── case class DynamicCodeApprovalConfigJsonV700( From 1b4030cf18dcc79ca97787d2a2f94ae7a06daf22 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 7 Sep 2026 10:31:35 +0200 Subject: [PATCH 07/13] Signal Channels documentation and tests --- obp-api/src/main/protobuf/signal.proto | 1 + .../SwaggerDefinitionsJSON.scala | 3 ++- .../main/scala/code/api/util/Glossary.scala | 26 +++++++++++++++++-- .../api/util/SelfServiceRateLimiter.scala | 3 ++- .../scala/code/api/v6_0_0/Http4s600.scala | 6 +++-- .../code/api/v6_0_0/JSONFactory6.0.0.scala | 6 ++++- .../code/obp/grpc/chat/AuthInterceptor.scala | 14 ++++++++++ .../signal/SignalChannelsServiceImpl.scala | 19 +++++++++----- .../obp/grpc/signal/api/FetchResponse.scala | 25 +++++++++++++++--- .../obp/grpc/signal/api/SignalProto.scala | 1 + .../scala/code/signal/SignalChannels.scala | 9 ++++++- .../code/api/v6_0_0/SignalChannelTest.scala | 3 +++ .../obp/grpc/SignalChannelsGrpcTest.scala | 7 +++++ 13 files changed, 105 insertions(+), 18 deletions(-) diff --git a/obp-api/src/main/protobuf/signal.proto b/obp-api/src/main/protobuf/signal.proto index 8c854a4e0d..4c9a951b6c 100644 --- a/obp-api/src/main/protobuf/signal.proto +++ b/obp-api/src/main/protobuf/signal.proto @@ -62,6 +62,7 @@ message FetchResponse { bool has_more = 4; int64 latest_sequence = 5; // newest message in the channel, 0 when empty int64 next_after_sequence = 6; // pass back as after_sequence to continue (advances past hidden private messages too) + int64 visible_count = 7; // messages in the channel the caller may see; total_count includes private ones hidden from them } // --- ListChannels: 1:1 with GET /signal-channels --- diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 40b823ccbb..8dc58bf264 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -6479,7 +6479,8 @@ object SwaggerDefinitionsJSON { total_count = 1, has_more = false, latest_sequence = 1771583400123456L, - next_after_sequence = 1771583400123456L + next_after_sequence = 1771583400123456L, + visible_count = 1 ) lazy val signalMessagePublishedJsonV600 = SignalMessagePublishedJsonV600( diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 1dcd6d8554..233724d6b7 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -515,7 +515,7 @@ object Glossary extends MdcLoggable { |- **consent_request** — Create Consent Request, Create Consent Request VRP |- **consumer_registration** — Create a Consumer (Dynamic Registration) |- **lookup** — Validate and check IBAN - |- **signal_channel_create** — Publish Signal Message, counted only when it creates a new channel (over gRPC this scope is keyed by Consumer instead) + |- **signal_channel_create** — Publish Signal Message, counted only when it creates a new channel, over REST and gRPC alike (gRPC uses the socket peer address; if none is available it falls back to the Consumer) | |Each scope has per-minute, per-hour and per-day limits per IP, with built-in defaults chosen so that a person or a well-behaved agent never reaches them, plus an optional global per-hour cap across all addresses that acts as a circuit breaker. Every request is counted, whether or not it succeeds. Counters live in Redis and fail open. | @@ -6172,6 +6172,28 @@ object Glossary extends MdcLoggable { |## Payloads are data, not instructions |Signal channels are readable and writable by any authenticated consumer on the instance. If your agent feeds received payloads to an LLM, treat them as **untrusted data, never as instructions** — the character checks above stop display-layer trickery, but no server-side check can stop a payload from *saying* something misleading. Prompt-injection defence belongs in the consuming agent. | + |## Conventions for agents that have never met + |Signal channels impose no protocol, and two agents written by different people will only find each other if they follow the same small habits. These are recommendations, not server rules. + | + |**Where to look first.** Announce yourself on the channel named `discovery` as soon as you have a token, then read `discovery` before anything else. Use `discovery` for presence and for finding a peer; move the actual conversation to a topic channel and name it in your announcement (`reply_channel`). An agent that only ever reads one fixed channel of its own choosing will miss peers who chose a different name; if `discovery` is empty, list the channels and read them all. + | + |**Message types.** Put the intent in `message_type` and keep the payload for content: + | + |- `announce` — "I exist": `agent_name`, `capabilities`, and the `reply_channel` you will read. + |- `hello` — a greeting addressed to whoever is listening, asking for a `reply`. + |- `reply` — an answer to a `hello` or any other message. + |- `ack` — "received", when the sender asked for confirmation; carries nothing new. + |- `proposal` — a list of `items` (each with `id`, `title`, `detail`, `proposed_owner`) for the other side to accept or change. + |- `counter` — the same list, edited; say in `text` what changed. + |- `agree` — the final list copied back verbatim, so both sides hold the same text. + |- `status` — progress on an agreed item: `items`, `state` (for example `approved`, `declined`, `done`), and who is acting. + | + |**Payload fields.** Always include `agent_name` (a human-readable name) and `from_user_id` (your OBP user id, so a peer can reply privately with `to_user_id`). When answering, include `in_reply_to` with the `message_id` or the `sequence` of the message you answer, and `reply_channel` when you want the answer somewhere else. Keep `text` for prose a human can read; put anything a program must parse in its own field. + | + |**Waiting and repeating.** Poll with `after_sequence`, not offset. If nothing arrives, do not repeat the same message; one `hello` is enough, and a peer that appears later reads the channel back. Messages expire with the channel, so a conversation that must survive an hour of silence belongs in Chat, not here. + | + |**Approval stays with people.** A `proposal` and an `agree` between agents settle what could be done and by whom; each agent still asks its own user before doing anything. Say so in the proposal, and post a `status` once the user has decided. + | |## Getting credentials as an agent (no account needed) |An agent does not need a pre-existing OBP user, consumer key or password. Where the instance runs OBP-OIDC with dynamic client registration enabled, three unauthenticated calls are enough: | @@ -6196,7 +6218,7 @@ object Glossary extends MdcLoggable { |## Endpoints |See the API Explorer tags **Signal-Channel** / **AI-Agent**: list channels, channel info, channel stats, publish message, get messages (offset/limit polling), delete channel — under `/obp/v6.0.0/signal-channels/...`. | - |Note on `total_count` in Get Signal Messages: it counts every message in the channel, including private messages hidden from the caller, so it can exceed the number of messages returned. Poll with `after_sequence` and `next_after_sequence` rather than comparing counts. + |Note on counts in Get Signal Messages: `total_count` counts every message in the channel, including private messages hidden from the caller, so it can exceed the number of messages returned; `visible_count` counts only the messages the caller may see and is the one to compare with what you have received. To detect newer messages, poll with `after_sequence` and `next_after_sequence` rather than comparing counts. | |## gRPC |The same operations are served over gRPC by `SignalChannelsService` (package `code.obp.grpc.signal.g1`, contract in `signal.proto`) when the gRPC server is enabled (`grpc.server.enabled`): **Publish**, **Fetch** and **ListChannels** are 1:1 with the REST endpoints and share their storage, and **Subscribe** is a server-side stream of new messages on one channel. Subscribe is live only — no catch-up, no replay — and applies the same privacy filter as Fetch. Each publish, REST or gRPC, is pushed to subscribers through Redis pub/sub. diff --git a/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala index ff8e6024c1..67cbcbb00b 100644 --- a/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala +++ b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala @@ -18,7 +18,8 @@ import net.liftweb.common.Full * (DirectLogin via AuthUser.getResourceUserId, DAuth, GatewayLogin, SIWE), so counting them * here too would count each attempt twice. Every self-service endpoint belongs to a * named *scope*; each scope has per-key limits (per minute, per hour, per day, keyed by the - * client IP address, or by consumer for gRPC) and an optional global per-hour cap that acts + * client IP address on REST and gRPC alike, with a Consumer fallback over gRPC when no peer + * address is available) and an optional global per-hour cap that acts * as a circuit breaker against a spray from many addresses. * * Props (all optional; built-in defaults apply): diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 5339090275..5040c496de 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -10582,8 +10582,10 @@ object Http4s600 { |Without `after_sequence`, offset and limit page over the channel's current contents. | |`total_count` is the number of messages in the channel including private messages hidden - |from you, so it can be larger than the number of messages returned. Do not use it to detect - |missed messages; use `after_sequence` and `next_after_sequence`. + |from you, so it can be larger than the number of messages returned. `visible_count` is the + |number you may see, over the whole channel, and is the one to compare with what you have + |received. Neither is a way to detect missed messages; use `after_sequence` and + |`next_after_sequence` for that. | |Authentication is Required. | diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 06e53e9159..96125d3258 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -1222,13 +1222,17 @@ case class SignalMessageJsonV600( case class SignalMessagesJsonV600( channel_name: String, messages: List[SignalMessageJsonV600], + // Every message in the channel, including private messages hidden from the caller. total_count: Long, has_more: Boolean, // Sequence of the newest message in the channel (0 when empty). latest_sequence: Long, // Pass this back as after_sequence to continue. It advances past messages the privacy // filter hid from you, so a page can be empty and the cursor still moves. - next_after_sequence: Long + next_after_sequence: Long, + // Messages in the channel the caller may see (broadcasts plus private messages to or from + // them). Compare this, not total_count, with what you have received. + visible_count: Long ) case class SignalMessagePublishedJsonV600( diff --git a/obp-api/src/main/scala/code/obp/grpc/chat/AuthInterceptor.scala b/obp-api/src/main/scala/code/obp/grpc/chat/AuthInterceptor.scala index 9f7aaafe49..da41a5ec64 100644 --- a/obp-api/src/main/scala/code/obp/grpc/chat/AuthInterceptor.scala +++ b/obp-api/src/main/scala/code/obp/grpc/chat/AuthInterceptor.scala @@ -31,6 +31,19 @@ class AuthInterceptor extends ServerInterceptor with MdcLoggable { import AuthInterceptor._ + /** The TCP peer of the gRPC call as a bare IP address, or "" when unavailable. This is the + * socket peer only: gRPC carries no trusted-proxy header handling here, so behind a + * forwarding proxy every caller shares the proxy's address. Used to populate + * CallContext.ipAddress so per-IP rate limiting keys the same way as REST. */ + private def peerIpAddress(call: ServerCall[_, _]): String = + try { + Option(call.getAttributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)) match { + case Some(inet: java.net.InetSocketAddress) => Option(inet.getAddress).map(_.getHostAddress).getOrElse(inet.getHostString) + case Some(other) => other.toString + case None => "" + } + } catch { case scala.util.control.NonFatal(_) => "" } + override def interceptCall[ReqT, RespT]( call: ServerCall[ReqT, RespT], headers: Metadata, @@ -59,6 +72,7 @@ class AuthInterceptor extends ServerInterceptor with MdcLoggable { // — not requestHeaders — to pick a scheme. val parsed = AuthHeaderParser.parseAuthorizationHeader(Some(authValue)) val cc = CallContext( + ipAddress = peerIpAddress(call), requestHeaders = List(HTTPParam("Authorization", List(authValue))), authReqHeaderField = parsed.authReqHeaderField, directLoginParams = parsed.directLoginParams, diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala index 8dd4c43f89..fcb286be2a 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala @@ -88,12 +88,18 @@ object SignalChannelsServiceImpl extends SignalChannelsServiceGrpc.SignalChannel message_type = Option(request.messageType).filter(_.nonEmpty), to_user_id = Option(request.toUserId).filter(_.nonEmpty)) val consumerId = callContext.flatMap(_.consumer match { case Full(c) => Some(c.consumerId.get); case _ => None }).getOrElse("") - // Channel creation cap (scope signal_channel_create). REST keys this on the client IP in - // SelfServiceRateLimitMiddleware; gRPC has no request IP in scope, so it keys on the - // Consumer (falling back to the User). Publishing to an existing channel is never counted. + // Channel creation cap (scope signal_channel_create), keyed by the client IP address exactly + // as SelfServiceRateLimitMiddleware keys it for REST, so one caller shares one counter on + // both transports. AuthInterceptor captures the peer address into CallContext.ipAddress; + // if none is available the key falls back to the Consumer, then the User. Publishing to an + // existing channel is never counted. if (RedisMessaging.channelInfo(request.channelName).isEmpty) { - val key = if (consumerId.nonEmpty) consumerId else user.userId - SelfServiceRateLimiter.check("signal_channel_create", key, "consumer") match { + val ip = callContext.map(_.ipAddress).getOrElse("").trim + val (key, keyKind) = + if (ip.nonEmpty && !ip.equalsIgnoreCase("unknown")) (ip, "ip") + else if (consumerId.nonEmpty) (consumerId, "consumer") + else (user.userId, "consumer") + SelfServiceRateLimiter.check("signal_channel_create", key, keyKind) match { case SelfServiceRateLimiter.Blocked(scope, _, exceeded) => throw Status.RESOURCE_EXHAUSTED .withDescription(SelfServiceRateLimiter.blockedMessage(scope, exceeded)) @@ -124,7 +130,8 @@ object SignalChannelsServiceImpl extends SignalChannelsServiceGrpc.SignalChannel totalCount = page.total_count, hasMore = page.has_more, latestSequence = page.latest_sequence, - nextAfterSequence = page.next_after_sequence) + nextAfterSequence = page.next_after_sequence, + visibleCount = page.visible_count) } override def listChannels(request: ListChannelsRequest): Future[ListChannelsResponse] = withUser { _ => diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala index ef69ee7980..b31bd7b0bd 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/FetchResponse.scala @@ -14,7 +14,8 @@ final case class FetchResponse( totalCount: _root_.scala.Long = 0L, hasMore: _root_.scala.Boolean = false, latestSequence: _root_.scala.Long = 0L, - nextAfterSequence: _root_.scala.Long = 0L + nextAfterSequence: _root_.scala.Long = 0L, + visibleCount: _root_.scala.Long = 0L ) extends scalapb.GeneratedMessage with scalapb.Message[FetchResponse] with scalapb.lenses.Updatable[FetchResponse] { @transient private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 @@ -26,6 +27,7 @@ final case class FetchResponse( if (hasMore != false) { __size += _root_.com.google.protobuf.CodedOutputStream.computeBoolSize(4, hasMore) } if (latestSequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(5, latestSequence) } if (nextAfterSequence != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(6, nextAfterSequence) } + if (visibleCount != 0L) { __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(7, visibleCount) } __size } final override def serializedSize: _root_.scala.Int = { @@ -47,6 +49,7 @@ final case class FetchResponse( { val __v = hasMore; if (__v != false) _output__.writeBool(4, __v) }; { val __v = latestSequence; if (__v != 0L) _output__.writeInt64(5, __v) }; { val __v = nextAfterSequence; if (__v != 0L) _output__.writeInt64(6, __v) }; + { val __v = visibleCount; if (__v != 0L) _output__.writeInt64(7, __v) }; } def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.signal.api.FetchResponse = { var __channelName = this.channelName @@ -55,6 +58,7 @@ final case class FetchResponse( var __hasMore = this.hasMore var __latestSequence = this.latestSequence var __nextAfterSequence = this.nextAfterSequence + var __visibleCount = this.visibleCount var _done__ = false while (!_done__) { val _tag__ = _input__.readTag() @@ -72,6 +76,8 @@ final case class FetchResponse( __latestSequence = _input__.readInt64() case 48 => __nextAfterSequence = _input__.readInt64() + case 56 => + __visibleCount = _input__.readInt64() case tag => _input__.skipField(tag) } } @@ -81,7 +87,8 @@ final case class FetchResponse( totalCount = __totalCount, hasMore = __hasMore, latestSequence = __latestSequence, - nextAfterSequence = __nextAfterSequence + nextAfterSequence = __nextAfterSequence, + visibleCount = __visibleCount ) } def withChannelName(__v: _root_.scala.Predef.String): FetchResponse = copy(channelName = __v) @@ -93,6 +100,7 @@ final case class FetchResponse( def withHasMore(__v: _root_.scala.Boolean): FetchResponse = copy(hasMore = __v) def withLatestSequence(__v: _root_.scala.Long): FetchResponse = copy(latestSequence = __v) def withNextAfterSequence(__v: _root_.scala.Long): FetchResponse = copy(nextAfterSequence = __v) + def withVisibleCount(__v: _root_.scala.Long): FetchResponse = copy(visibleCount = __v) def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { (__fieldNumber: @_root_.scala.unchecked) match { case 1 => { @@ -116,6 +124,10 @@ final case class FetchResponse( val __t = nextAfterSequence if (__t != 0L) __t else null } + case 7 => { + val __t = visibleCount + if (__t != 0L) __t else null + } } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -127,6 +139,7 @@ final case class FetchResponse( case 4 => _root_.scalapb.descriptors.PBoolean(hasMore) case 5 => _root_.scalapb.descriptors.PLong(latestSequence) case 6 => _root_.scalapb.descriptors.PLong(nextAfterSequence) + case 7 => _root_.scalapb.descriptors.PLong(visibleCount) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -144,7 +157,8 @@ object FetchResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.sig __fieldsMap.getOrElse(__fields.get(2), 0L).asInstanceOf[_root_.scala.Long], __fieldsMap.getOrElse(__fields.get(3), false).asInstanceOf[_root_.scala.Boolean], __fieldsMap.getOrElse(__fields.get(4), 0L).asInstanceOf[_root_.scala.Long], - __fieldsMap.getOrElse(__fields.get(5), 0L).asInstanceOf[_root_.scala.Long] + __fieldsMap.getOrElse(__fields.get(5), 0L).asInstanceOf[_root_.scala.Long], + __fieldsMap.getOrElse(__fields.get(6), 0L).asInstanceOf[_root_.scala.Long] ) } implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.signal.api.FetchResponse] = _root_.scalapb.descriptors.Reads{ @@ -156,7 +170,8 @@ object FetchResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.sig __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Long]).getOrElse(0L), __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Boolean]).getOrElse(false), __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Long]).getOrElse(0L), - __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).map(_.as[_root_.scala.Long]).getOrElse(0L) + __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).map(_.as[_root_.scala.Long]).getOrElse(0L), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Long]).getOrElse(0L) ) case _ => throw new RuntimeException("Expected PMessage") } @@ -180,6 +195,7 @@ object FetchResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.sig def hasMore: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Boolean] = field(_.hasMore)((c_, f_) => c_.copy(hasMore = f_)) def latestSequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.latestSequence)((c_, f_) => c_.copy(latestSequence = f_)) def nextAfterSequence: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.nextAfterSequence)((c_, f_) => c_.copy(nextAfterSequence = f_)) + def visibleCount: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.visibleCount)((c_, f_) => c_.copy(visibleCount = f_)) } final val CHANNEL_NAME_FIELD_NUMBER = 1 final val MESSAGES_FIELD_NUMBER = 2 @@ -187,4 +203,5 @@ object FetchResponse extends scalapb.GeneratedMessageCompanion[code.obp.grpc.sig final val HAS_MORE_FIELD_NUMBER = 4 final val LATEST_SEQUENCE_FIELD_NUMBER = 5 final val NEXT_AFTER_SEQUENCE_FIELD_NUMBER = 6 + final val VISIBLE_COUNT_FIELD_NUMBER = 7 } diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala index cb81ff5b85..09b361d318 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/api/SignalProto.scala @@ -71,6 +71,7 @@ object SignalProto { .addField(boolField("has_more", 4)) .addField(int64Field("latest_sequence", 5)) .addField(int64Field("next_after_sequence", 6)) + .addField(int64Field("visible_count", 7)) ) // 6: ListChannelsRequest .addMessageType(DescriptorProto.newBuilder() diff --git a/obp-api/src/main/scala/code/signal/SignalChannels.scala b/obp-api/src/main/scala/code/signal/SignalChannels.scala index 4607d7df5b..ef2bbfe9da 100644 --- a/obp-api/src/main/scala/code/signal/SignalChannels.scala +++ b/obp-api/src/main/scala/code/signal/SignalChannels.scala @@ -74,7 +74,14 @@ object SignalChannels { private def page(channelName: String, raw: List[String], total: Long, hasMore: Boolean, latest: Long, nextAfter: Long, userId: String): SignalMessagesJsonV600 = { val visible = raw.flatMap(parseMessage).filter(isVisibleTo(_, userId)) - SignalMessagesJsonV600(channelName, visible, total, hasMore, latest, nextAfter) + SignalMessagesJsonV600(channelName, visible, total, hasMore, latest, nextAfter, visibleCount(channelName, userId)) + } + + /** How many messages in the channel the caller may see. Counted over the whole channel, not + * the page, so it is comparable with total_count. One extra Redis read per fetch. */ + def visibleCount(channelName: String, userId: String): Long = { + val (raw, _) = RedisMessaging.fetchMessages(channelName, 0, RedisMessaging.channelMaxMessages) + raw.flatMap(parseMessage).count(isVisibleTo(_, userId)).toLong } /** Channels holding at least one broadcast message. Private-only channels are not listed. */ diff --git a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala index 426ca37d9d..d3ab45443b 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/SignalChannelTest.scala @@ -112,6 +112,9 @@ class SignalChannelTest extends V600ServerSetup { val sequences = (messages \ "sequence").extract[List[Long]] sequences.forall(_ > firstSeq) should equal(true) (newer.body \ "has_more").extract[Boolean] should equal(false) + // Counts describe the whole channel, not the page: three messages, all broadcasts. + (newer.body \ "total_count").extract[Long] should equal(3L) + (newer.body \ "visible_count").extract[Long] should equal(3L) val nextAfter = (newer.body \ "next_after_sequence").extract[Long] nextAfter should equal((newer.body \ "latest_sequence").extract[Long]) diff --git a/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala b/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala index 1f471c1f1a..e841d6cdd8 100644 --- a/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala +++ b/obp-api/src/test/scala/code/obp/grpc/SignalChannelsGrpcTest.scala @@ -136,6 +136,8 @@ class SignalChannelsGrpcTest extends ServerSetupWithTestData { val seenBySender = publisher.fetch(FetchRequest(channelName, 0, 10)) seenBySender.totalCount should equal(3L) + // The sender sees everything it sent, so visible_count matches total_count for it. + seenBySender.visibleCount should equal(3L) // Sequences are stamped, strictly increasing, and reported consistently. broadcast.sequence should be > 0L privateMsg.sequence should be > broadcast.sequence @@ -161,9 +163,14 @@ class SignalChannelsGrpcTest extends ServerSetupWithTestData { val seenByRecipient = blockingStub(tokenOf(user2)).fetch(FetchRequest(channelName, 0, 10)) seenByRecipient.messages.map(_.messageId).toSet should equal(Set(broadcast.messageId, privateMsg.messageId)) + // total_count still counts the message to someone else; visible_count does not. + seenByRecipient.totalCount should equal(3L) + seenByRecipient.visibleCount should equal(2L) val seenByStranger = blockingStub(tokenOf(user3)).fetch(FetchRequest(channelName, 0, 10)) seenByStranger.messages.map(_.messageId) should equal(Seq(broadcast.messageId)) + seenByStranger.totalCount should equal(3L) + seenByStranger.visibleCount should equal(1L) publisher.listChannels(ListChannelsRequest()).channels.map(_.channelName) should contain(channelName) From 21419f1fc151f75d0865bf79a668a769c3143c77 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 7 Sep 2026 11:30:44 +0200 Subject: [PATCH 08/13] Update SignalChannelsServiceImpl.scala --- .../signal/SignalChannelsServiceImpl.scala | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala index fcb286be2a..8147fe522a 100644 --- a/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala +++ b/obp-api/src/main/scala/code/obp/grpc/signal/SignalChannelsServiceImpl.scala @@ -13,7 +13,7 @@ import com.google.protobuf.timestamp.Timestamp import com.openbankproject.commons.ExecutionContext.Implicits.global import com.openbankproject.commons.model.User import com.openbankproject.commons.util.JsonAliases -import io.grpc.{Status, StatusRuntimeException} +import io.grpc.{Context, Status, StatusRuntimeException} import io.grpc.stub.{ServerCallStreamObserver, StreamObserver} import net.liftweb.common.Full import org.json4s.JsonAST.JValue @@ -48,11 +48,19 @@ object SignalChannelsServiceImpl extends SignalChannelsServiceGrpc.SignalChannel private def withUser[T](body: User => T): Future[T] = { val user = AuthInterceptor.USER_CONTEXT_KEY.get() if (user == null) Future.failed(unauthenticated) - else Future(body(user)).recoverWith { - case e: StatusRuntimeException => Future.failed(e) - case NonFatal(e) => - logger.error(s"SignalChannelsServiceImpl says: ${e.getMessage}", e) - Future.failed(Status.INTERNAL.withDescription(e.getMessage).asRuntimeException()) + else { + // AuthInterceptor stores the User and the CallContext as gRPC Context values, which are + // thread-local. body runs in a Future on a different thread, so it must run *inside* the + // captured gRPC Context for AuthInterceptor.CALL_CONTEXT_KEY.get() to see the caller's + // context (consumer, ipAddress) rather than null. Without this the channel-creation rate + // limit keyed on an empty consumer/IP and fell back to the user id. + val grpcContext = Context.current() + Future(grpcContext.call(() => body(user))).recoverWith { + case e: StatusRuntimeException => Future.failed(e) + case NonFatal(e) => + logger.error(s"SignalChannelsServiceImpl says: ${e.getMessage}", e) + Future.failed(Status.INTERNAL.withDescription(e.getMessage).asRuntimeException()) + } } } From bf89e8ed8b46870587337bb16364f113f1719676 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 7 Sep 2026 18:48:18 +0200 Subject: [PATCH 09/13] New path for api tags in v7.0.0 + longer description for dynamic resource doc --- .../code/api/util/migration/Migration.scala | 9 +++ ...OfDynamicResourceDocTextFieldsLength.scala | 78 +++++++++++++++++++ .../scala/code/api/v4_0_0/Http4s400.scala | 17 ++++ .../scala/code/api/v5_1_0/Http4s510.scala | 6 +- .../scala/code/api/v7_0_0/Http4s700.scala | 37 +++++++++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 36 +++++++++ .../DynamicResourceDoc.scala | 6 +- .../scala/code/api/v7_0_0/ApiTagsTest.scala | 64 +++++++++++++++ 8 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/migration/MigrationOfDynamicResourceDocTextFieldsLength.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/ApiTagsTest.scala diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index 9a094352ef..604a58a104 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -161,6 +161,7 @@ object Migration extends MdcLoggable { migrateMetricCertificateTrust(startedBeforeSchemifier) dropFastFirehoseAccountsViews(startedBeforeSchemifier) alterDynamicResourceDocBodyFieldsLength() + alterDynamicResourceDocTextFieldsLength() } /** @@ -522,6 +523,14 @@ object Migration extends MdcLoggable { } } + // description / tags / roles of a Dynamic Resource Doc: varchar(255) -> text (see the migration object). + private def alterDynamicResourceDocTextFieldsLength(): Boolean = { + val name = nameOf(alterDynamicResourceDocTextFieldsLength) + runOnce(name) { + MigrationOfDynamicResourceDocTextFieldsLength.alterColumnsType(name) + } + } + private def dropFastFirehoseAccountsViews(startedBeforeSchemifier: Boolean): Boolean = { if(startedBeforeSchemifier == true) { logger.warn(s"Migration.database.dropFastFirehoseAccountsViews(true) cannot be run before Schemifier.") diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfDynamicResourceDocTextFieldsLength.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfDynamicResourceDocTextFieldsLength.scala new file mode 100644 index 0000000000..d3104e8f9f --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfDynamicResourceDocTextFieldsLength.scala @@ -0,0 +1,78 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import code.dynamicResourceDoc.DynamicResourceDoc +import code.util.Helper.MdcLoggable +import net.liftweb.common.Full +import net.liftweb.mapper.Schemifier + +/** + * description, tags and roles of a Dynamic Resource Doc were varchar(255). A prose description written + * for the API Explorer, or a comma-joined list of roles, routinely exceeds that and the create failed with + * a bare "value too long for type character varying(255)". The mapper now declares them as text; this + * widens the columns on databases created before that change. + * + * Column names are taken from the mapper rather than spelled out: Lift appends `_c` to any field whose + * name is a reserved word, so `Roles` is stored as `roles_c`, not `roles`. + * + * The migration is recorded as successful only when every ALTER actually ran. A failure is logged with + * isSuccessful = false and the exception text, and boot continues; because runOnce skips a script only + * once it is logged as executed, a failed run is retried on the next start. + */ +object MigrationOfDynamicResourceDocTextFieldsLength extends MdcLoggable { + def alterColumnsType(name: String): Boolean = { + DbFunction.tableExists(DynamicResourceDoc) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + var isSuccessful = false + val sqlLog = new StringBuilder + + val table = DynamicResourceDoc._dbTableNameLC + val columns = List( + DynamicResourceDoc.Description.dbColumnName, + DynamicResourceDoc.Tags.dbColumnName, + DynamicResourceDoc.Roles.dbColumnName + ) + val isSqlServer = APIUtil.getPropsValue("db.driver") match { + case Full(dbDriver) => dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") + case _ => false + } + val statements: List[String] = columns.map { column => + if (isSqlServer) s"ALTER TABLE $table ALTER COLUMN $column VARCHAR(MAX);" + else s"ALTER TABLE $table ALTER COLUMN $column TYPE text;" + } + + try { + statements.foreach { statement => + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => statement)).append("\n") + } + isSuccessful = true + } catch { + case e: Exception => + isSuccessful = false + sqlLog.append(s"\nException: ${e.getMessage}\n") + logger.error(s"Migration.database.$name failed: ${e.getMessage}", e) + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Executed SQL: + |$sqlLog + |""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = + s"""${DynamicResourceDoc._dbTableNameLC} table does not exist""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } +} diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 9a9652b66c..28e646bcda 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9489,9 +9489,26 @@ object Http4s400 { case _ => true } } + // Column widths: say which field is too long instead of letting the database answer OBP-50000. + _ <- checkDynamicResourceDocFieldLengths(body, cc) } yield () } + private val dynamicResourceDocMaxLengths: List[(String, JsonDynamicResourceDoc => String, Int)] = List( + ("partial_function_name", _.partialFunctionName, 255), + ("request_verb", _.requestVerb, 255), + ("request_url", _.requestUrl, 255), + ("summary", _.summary, 255), + ("description", _.description, 2000) + ) + + private def checkDynamicResourceDocFieldLengths(body: JsonDynamicResourceDoc, cc: CallContext): Future[Unit] = { + val tooLong = dynamicResourceDocMaxLengths.collect { + case (field, read, max) if Option(read(body)).exists(_.length > max) => s"$field must be at most $max characters (got ${read(body).length})" + } + code.util.Helper.booleanToFuture(s"$InvalidJsonFormat ${tooLong.mkString("; ")}", cc = Some(cc)) { tooLong.isEmpty }.map(_ => ()) + } + private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index fc65fa3ce8..233ee0aaea 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -1917,13 +1917,15 @@ object Http4s510 { "GET", "/tags", "Get API Tags", - s"""Get API TagsGet API Tags + s"""Returns the names of every API tag known to this instance (static tags and dynamic tags). + | + |For per-tag endpoint counts use `GET /obp/v7.0.0/api/tags` instead. | |${userAuthenticationMessage(false)} | |""", EmptyBody, - accountsMinimalJson400, + APITags(List("Account", "Bank", "Transaction Request")), List(UnknownError), List(apiTagApi), None, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 215f6e92f2..2e7c5dbf94 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -901,6 +901,43 @@ object Http4s700 { http4sPartialFunction = Some(getErrorMessages) ) + // Route: GET /obp/v7.0.0/api/tags + val getApiTags: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "api" / "tags" => + EndpointHelpers.executeAndRespond(req) { _ => + Future.successful(JSONFactory700.createApiTagsJsonV700(allResourceDocs.toList)) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getApiTags), + "GET", + "/api/tags", + "Get API Tags", + """Returns every API tag known to this instance, with the number of endpoints that carry each tag. + | + |Tags are the groupings used by API Explorer and the Resource Docs (e.g. `Account`, `Bank`, + |`Transaction Request`). The counts are taken from the aggregated v7.0.0 Resource Docs, i.e. the + |same set of endpoints returned by `GET /obp/v7.0.0/resource-docs/v7.0.0/obp` before any locale + |or content filtering. Dynamic tags (from Dynamic Entities and Dynamic Endpoints) are included + |and may have a count of 0. + | + |An endpoint with several tags is counted once under each of its tags, so the per-tag counts sum + |to more than `number_of_endpoints`, which is the number of distinct endpoints counted. + | + |`tags` is sorted by `number_of_endpoints` descending, then by tag name. + | + |This supersedes `GET /tags` (v5.1.0), which returns tag names only. + | + |No Authentication is Required.""".stripMargin, + EmptyBody, + JSONFactory700.apiTagsJsonV700Example, + List(UnknownError), + apiTagDocumentation :: apiTagApi :: Nil, + http4sPartialFunction = Some(getApiTags) + ) + // ── Phase 1 batch 2 ───────────────────────────────────────────────────── // Route: GET /obp/v7.0.0/users/user-id/USER_ID diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index eecf017b40..f7cbdf5157 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -161,6 +161,42 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { case class ErrorMessageEntryJsonV700(code: String, name: String, message: String) + // ─── API tags (GET /api/tags) ───────────────────────────────────────────────────────────── + /** One API tag and the number of endpoints in the aggregated v7.0.0 resource docs that carry it. */ + case class ApiTagJsonV700(tag: String, number_of_endpoints: Int) + + /** + * All API tags with per-tag endpoint counts, sorted by number_of_endpoints descending then tag name. + * `number_of_endpoints` at the top level is the number of distinct endpoints counted; an endpoint + * with several tags is counted once under each of them, so the per-tag counts sum to more than that. + */ + case class ApiTagsJsonV700(tags: List[ApiTagJsonV700], number_of_endpoints: Int) + + val apiTagsJsonV700Example: ApiTagsJsonV700 = ApiTagsJsonV700( + tags = List( + ApiTagJsonV700(tag = "Account", number_of_endpoints = 42), + ApiTagJsonV700(tag = "Bank", number_of_endpoints = 17), + ApiTagJsonV700(tag = "Transaction Request", number_of_endpoints = 12) + ), + number_of_endpoints = 900 + ) + + /** + * Counts endpoints per tag over the given resource docs and merges the result with every tag known to + * `ApiTag` (static and dynamic), so tags with no endpoints still appear with a count of 0. + */ + def createApiTagsJsonV700(resourceDocs: Seq[APIUtil.ResourceDoc]): ApiTagsJsonV700 = { + val counts: Map[String, Int] = resourceDocs + .flatMap(_.tags.map(_.displayTag).distinct) + .groupBy(identity) + .map { case (tag, occurrences) => tag -> occurrences.size } + val allTagNames: Set[String] = code.api.util.ApiTag.allDisplayTagNames ++ counts.keySet + val tags = allTagNames.toList + .map(tag => ApiTagJsonV700(tag, counts.getOrElse(tag, 0))) + .sortBy(t => (-t.number_of_endpoints, t.tag)) + ApiTagsJsonV700(tags, resourceDocs.size) + } + // ─── Rate limiter config (GET /management/rate-limiter-config) ───────────────────────── /** One limit row of a rate limiter. Windows the limiter does not have are absent; -1 means unlimited, 0 blocks. */ case class RateLimiterLimitJsonV700( diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index f15438ca2a..7f6eb86d85 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -18,12 +18,12 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object RequestVerb extends MappedString(this, 255) object RequestUrl extends MappedString(this, 255) object Summary extends MappedString(this, 255) - object Description extends MappedString(this, 255) + object Description extends MappedText(this) object ExampleRequestBody extends MappedText(this) object SuccessResponseBody extends MappedText(this) object ErrorResponseBodies extends MappedText(this) - object Tags extends MappedString(this, 255) - object Roles extends MappedString(this, 255) + object Tags extends MappedText(this) + object Roles extends MappedText(this) object MethodBody extends MappedText(this) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext diff --git a/obp-api/src/test/scala/code/api/v7_0_0/ApiTagsTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/ApiTagsTest.scala new file mode 100644 index 0000000000..06e8d500cb --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/ApiTagsTest.scala @@ -0,0 +1,64 @@ +package code.api.v7_0_0 + +import code.api.v7_0_0.Http4s700.Implementations7_0_0 +import code.api.v7_0_0.JSONFactory700.ApiTagsJsonV700 +import code.setup.ServerSetupWithTestData +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * GET /obp/v7.0.0/api/tags — every API tag with the number of endpoints that carry it. + * + * The counts come from Http4s700.allResourceDocs (all versions, deduplicated), so the test + * cross-checks one tag against that same source rather than against a hard-coded number. + */ +class ApiTagsTest extends ServerSetupWithTestData { + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object GetApiTags extends Tag(nameOf(Implementations7_0_0.getApiTags)) + + def v7 = baseRequest / "obp" / "v7.0.0" + + feature(s"test ${GetApiTags}") { + scenario("anonymous GET returns every tag with an endpoint count", GetApiTags, VersionOfApi) { + When("we call the endpoint without authentication") + val response = makeGetRequest(v7 / "api" / "tags") + + Then("it is public and returns 200") + response.code should equal(200) + val json = response.body.extract[ApiTagsJsonV700] + + And("there are tags and endpoints") + json.tags should not be empty + json.number_of_endpoints should be > 0 + json.number_of_endpoints should equal(Http4s700.allResourceDocs.size) + + And("well-known tags carry endpoints") + val byTag = json.tags.map(t => t.tag -> t.number_of_endpoints).toMap + byTag("Account") should be > 0 + byTag("Bank") should be > 0 + + And("a tag's count matches the aggregated resource docs") + val expectedAccountCount = Http4s700.allResourceDocs.count(_.tags.exists(_.displayTag == "Account")) + byTag("Account") should equal(expectedAccountCount) + + And("no count is negative and tags are unique") + json.tags.foreach(_.number_of_endpoints should be >= 0) + json.tags.map(_.tag).distinct.size should equal(json.tags.size) + + And("tags are sorted by count descending then by name") + val counts = json.tags.map(_.number_of_endpoints) + counts should equal(counts.sorted.reverse) + json.tags.sortBy(t => (-t.number_of_endpoints, t.tag)) should equal(json.tags) + } + + scenario("the v5.1.0 GET /tags name list is a subset of the v7.0.0 tags", GetApiTags, VersionOfApi) { + val v7Tags = makeGetRequest(v7 / "api" / "tags").body.extract[ApiTagsJsonV700].tags.map(_.tag).toSet + val v51 = makeGetRequest(baseRequest / "obp" / "v5.1.0" / "tags") + v51.code should equal(200) + val v51Tags = v51.body.extract[code.api.v5_1_0.APITags].tags.toSet + v51Tags.subsetOf(v7Tags) shouldBe true + } + } +} From 550d7c0edc2e955b29e9781ba47cbf382c5f0c52 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 8 Sep 2026 05:13:10 +0200 Subject: [PATCH 10/13] applyRateLimiting once instead of on each fall through from v7.0.0 -> v6.0.0 -> v5.1.0 etc. --- .../main/scala/code/api/util/APIUtil.scala | 31 ++++++-- .../code/api/util/http4s/Http4sApp.scala | 10 ++- .../code/api/util/http4s/Http4sSupport.scala | 19 +++++ .../util/http4s/ResourceDocMiddleware.scala | 78 ++++++++++++++++--- .../code/api/v6_0_0/RateLimitsTest.scala | 35 +++++++++ 5 files changed, 155 insertions(+), 18 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index adec0652fd..64058b0ee9 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -3096,8 +3096,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * user and session, verify the signed request, run the Berlin Group checks and apply rate * limiting. Each caller decides on its own what to make of the outcome. * @param cc The call context of an request + * @param applyRateLimiting false skips the rate-limit step (the call is neither refused nor + * counted). Only [[resolveCallerWithoutRateLimiting]] passes false. */ - private def accessPipeline(cc: CallContext): OBPReturnType[Box[User]] = { + private def accessPipeline(cc: CallContext, applyRateLimiting: Boolean = true): OBPReturnType[Box[User]] = { getUserAndSessionContextFuture(cc) map { result => val (body, verb, url, reqHeaders) = requestPartsOf(result) // Verify signed request @@ -3108,14 +3110,33 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ BerlinGroupCheck.validate(body, verb, url, reqHeaders, result) } map { result => - val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root,getOAuth2ServerWellKnown").split(",").toList - cc.resourceDocument.map(_.partialFunctionName) match { - case Some(functionName) if excludeFunctions.exists(_ == functionName) => result - case _ => RateLimitingUtil.underCallLimits(result) + if (!applyRateLimiting) result + else { + val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root,getOAuth2ServerWellKnown").split(",").toList + cc.resourceDocument.map(_.partialFunctionName) match { + case Some(functionName) if excludeFunctions.exists(_ == functionName) => result + case _ => RateLimitingUtil.underCallLimits(result) + } } } } + /** + * Resolve the caller (user and Consumer) from the request credentials WITHOUT applying rate + * limiting: the call is neither refused for exceeding a limit nor counted against one. + * + * For the http4s version fallthrough chain only (ResourceDocMiddleware.resolveCallerOnce). A hop + * that has no ResourceDoc for the request is almost always about to pass it on to the next + * version, and the hop that finally serves it applies rate limiting itself through + * [[anonymousAccess]] / [[applicationAccess]]. Counting the call on every hop charged one unit per + * hop: `GET /obp/v5.1.0/users/current` is served by v3.0.0 after six hops, so it cost seven + * units and a Consumer with a per-second limit below seven could never call it (429 OBP-10018). + * + * A Failure is returned in the Box, never thrown; the middleware decides what to do with it. + */ + def resolveCallerWithoutRateLimiting(cc: CallContext): OBPReturnType[Box[User]] = + accessPipeline(cc, applyRateLimiting = false) + /** * This function is used to introduce Rate Limit at an unauthorized endpoint * @param cc The call context of an request diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala index 8867f8911f..a975bf39e1 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala @@ -118,8 +118,16 @@ object Http4sApp extends MdcLoggable { } } + // One empty holder per incoming request for the caller resolved on a no-ResourceDoc hop, shared by + // every link of the fallthrough chain below (see ResourceDocMiddleware.resolveCallerOnce). Bridges + // rewrite the URI between hops with `req.withUri`, which keeps attributes, so the holder travels + // the chain and dies with the request. + private def installCallerResolvedOnThisRequest(req: Request[IO]): Request[IO] = + if (req.attributes.lookup(Http4sRequestAttributes.callerResolvedOnThisRequestKey).isDefined) req + else req.withAttribute(Http4sRequestAttributes.callerResolvedOnThisRequestKey, Http4sRequestAttributes.newCallerResolvedOnThisRequest) + private def baseServices: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req: Request[IO] => - OptionT.liftF(cacheBodyOnce(req)).flatMap { req => + OptionT.liftF(cacheBodyOnce(req).map(installCallerResolvedOnThisRequest)).flatMap { req => corsHandler.run(req) .orElse(AppsPage.routes.run(req)) .orElse(StatusPage.routes.run(req)) diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index 15faa0f102..dddec1463d 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -79,6 +79,25 @@ object Http4sRequestAttributes { val callerCertificateTrustKey: Key[code.api.util.PeerTrust.Resolution] = Key.newKey[IO, code.api.util.PeerTrust.Resolution].unsafeRunSync()(cats.effect.unsafe.IORuntime.global) + /** Outcome of resolving the caller from the request credentials: the user (or a Failure) and the + * CallContext enriched by that resolution (Consumer, session, rate-limit config, ...). */ + type ResolvedCaller = (Box[User], Option[CallContext]) + + /** + * The caller resolved on THIS request by a fallthrough hop that has no ResourceDoc for it (see + * ResourceDocMiddleware.resolveCallerOnce). Http4sApp attaches one empty holder to each incoming + * request; the first such hop fills it and every later link of the version fallthrough chain + * reads it instead of re-validating the credentials (JWKS lookups, Consumer lookup and save). + * The holder is a request attribute: it lives in memory for the duration of that one request, + * is never stored anywhere else and is unreachable from any other request. When it is absent + * (e.g. the middleware used on its own in a test) the middleware still resolves, just on every hop. + */ + val callerResolvedOnThisRequestKey: Key[java.util.concurrent.atomic.AtomicReference[Option[ResolvedCaller]]] = + Key.newKey[IO, java.util.concurrent.atomic.AtomicReference[Option[ResolvedCaller]]].unsafeRunSync()(cats.effect.unsafe.IORuntime.global) + + def newCallerResolvedOnThisRequest: java.util.concurrent.atomic.AtomicReference[Option[ResolvedCaller]] = + new java.util.concurrent.atomic.AtomicReference[Option[ResolvedCaller]](None) + /** * Implicit class that adds .callContext accessor to Request[IO]. diff --git a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala index d2adad6e8d..888f87a239 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala @@ -173,25 +173,79 @@ object ResourceDocMiddleware extends MdcLoggable { OptionT(work.timeoutTo(endpointTimeoutMs.millis, endpointTimeoutResponse(req))) case None => - // No matching ResourceDoc: fallback to original route (NO transaction scope opened). - // Attach the basic CC so req.callContext works in the inner route even without a doc match. - // Carry the cached body forward so the bridge cascade can still read it. - // Best-effort authentication: populate cc.user from request credentials so that - // withUser/withUserAndBank handlers return 401/403 correctly (e.g. empty path segments - // that bypass ResourceDocMatcher but still match a route pattern). + // This group has no ResourceDoc for the request. Almost always the request is simply + // not ours: `routes.run` yields None and the request moves to the next link of the + // version fallthrough chain (v7 -> v6 -> v5.1 -> bridges -> ... ). No transaction scope + // is opened. The cached body is carried forward for the later hops. + // + // The one case where the inner routes DO serve such a request is a malformed URL with + // an empty path segment (e.g. `/banks//accounts`): the matcher counts segments and finds + // no doc, but the http4s pattern still matches with an empty id, and its handler needs + // the caller in the CallContext to answer 403/404 rather than a misleading 401. That is + // why the caller is resolved here at all. It is resolved WITHOUT rate limiting and at + // most ONCE per request (see resolveCallerOnce): rate limiting belongs to the hop that + // serves the request, and re-validating the credentials on every hop was pure waste. OptionT.liftF( - IO.fromFuture(IO(APIUtil.anonymousAccess(cc))).map { - case (Full(user), Some(updatedCC)) => reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, updatedCC.copy(user = Full(user))) - case (Full(user), None) => reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, cc.copy(user = Full(user))) - case (_, Some(updatedCC)) => reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, updatedCC) - case _ => reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, cc) - }.recover { case _ => reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, cc) } + resolveCallerOnce(req, cc).map { resolvedCc => + reqWithCachedBody.withAttribute(Http4sRequestAttributes.callContextKey, resolvedCc) + } ).flatMap(routes.run) } } } } + /** + * Resolve the caller for a hop that has no ResourceDoc for the request, once per request. + * + * The first such hop runs [[APIUtil.resolveCallerWithoutRateLimiting]] and stores the outcome in + * the holder Http4sApp attached to the request (`Http4sRequestAttributes.callerResolvedOnThisRequestKey`); + * every later hop of the same request reads it. Before this, each hop authenticated afresh AND counted the call + * against the Consumer's rate limit: `GET /obp/v5.1.0/users/current` crossed seven hops before + * v3.0.0 served it, so it validated the token seven times, saved the Consumer row seven times, + * and cost seven rate-limit units - a Consumer limited to fewer than seven calls per second was + * refused with 429 OBP-10018 on the second hop. + * + * Failures (bad token, unknown consumer, ...) are kept too - they are the outcome for this + * request - and leave the CallContext without a user, exactly as before. Only an exception + * thrown by the pipeline is not kept; the hop then proceeds with the unresolved context. + */ + private def resolveCallerOnce(req: Request[IO], cc: CallContext): IO[CallContext] = { + val holder = req.attributes.lookup(Http4sRequestAttributes.callerResolvedOnThisRequestKey) + holder.flatMap(_.get()) match { + case Some(resolved) => + IO.pure(withResolvedCaller(cc, resolved)) + case None => + IO.fromFuture(IO(APIUtil.resolveCallerWithoutRateLimiting(cc))).attempt.map { + case Right(resolved) => + holder.foreach(_.set(Some(resolved))) + withResolvedCaller(cc, resolved) + case Left(NonFatal(e)) => + logger.debug(s"[ResourceDocMiddleware] caller resolution threw on a no-ResourceDoc hop for ${req.method.name} ${req.uri.path.renderString}: ${e.getMessage}") + cc + case Left(e) => throw e + } + } + } + + /** + * Merge a resolution into THIS hop's freshly built CallContext. The resolved context may come + * from an earlier hop; the bridges rewrite the path between hops (`/obp/v5.1.0/...` -> + * `/obp/v5.0.0/...`), so the current hop's `url` and `implementedInVersion` are kept while the + * authentication-derived fields (user, consumer, session, rate-limit config, ...) are taken from + * the resolution. + */ + private def withResolvedCaller(cc: CallContext, resolved: Http4sRequestAttributes.ResolvedCaller): CallContext = { + def carryOver(resolvedCc: CallContext): CallContext = + resolvedCc.copy(url = cc.url, implementedInVersion = cc.implementedInVersion) + resolved match { + case (Full(user), Some(resolvedCc)) => carryOver(resolvedCc).copy(user = Full(user)) + case (Full(user), None) => cc.copy(user = Full(user)) + case (_, Some(resolvedCc)) => carryOver(resolvedCc) + case _ => cc + } + } + /** 504 response emitted when endpointTimeoutMs elapses before the handler completes. */ private def endpointTimeoutResponse(req: Request[IO]): IO[Option[Response[IO]]] = IO { logger.warn( diff --git a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala index 65d63e432b..17b89caaf4 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala @@ -27,6 +27,8 @@ package code.api.v6_0_0 import org.json4s._ import code.api.util.APIUtil.OAuth._ +import code.api.Constant +import code.api.cache.Redis import code.api.util.ApiRole.{CanCreateRateLimits, CanDeleteRateLimits, CanGetRateLimits} import code.api.util.ErrorMessages.{UserHasMissingRoles, AuthenticatedUserIsRequired, TooManyRequests} import code.api.v6_0_0.Http4s600.Implementations6_0_0 @@ -387,6 +389,39 @@ class RateLimitsTest extends V600ServerSetup { callAsUser3().code should equal(200) } + // GET /obp/v5.1.0/users/current has no v5.1.0 ResourceDoc, so the request passes the v7.0.0, v6.0.0 + // and v5.1.0 groups (three hops), then the v5.1.0 -> v5.0.0 -> v4.0.0 -> v3.1.0 -> v3.0.0 bridges, + // and v3.0.0 serves it: seven hops. Every hop that had no doc used to authenticate afresh AND + // charge one rate-limit unit, so one request cost seven units and a per-minute limit of 2 + // refused the very first request with 429 OBP-10018. A request must cost exactly one unit. + scenario("A request served after six version hops costs one rate-limit unit, not one per hop", ApiEndpoint4, VersionOfApi) { + Given("A record limiting the consumer to 2 calls per minute, unlimited otherwise") + // Earlier scenarios in this class called the API as user3 within the same minute, and + // counters are incremented even under an unlimited record: start this window from zero. + Redis.deleteKeysByPattern(s"${Constant.CALL_COUNTER_PREFIX}${consumerId3}_*") + val id = createLimit(consumerId3, callLimitJson("-1", "2", "-1")) + try { + def callV510AsUser3() = makeGetRequest((v5_1_0_Request / "users" / "current").GET <@ (user3)) + When("The consumer makes a request that v3.0.0 serves after six version hops") + val first = callV510AsUser3() + Then("it is served, and by a bridged version") + first.code should equal(200) + first.headers.flatMap(h => Option(h.get("X-OBP-Version-Served"))) should not be empty + And("a second request is served too: the first one cost one unit, not seven") + callV510AsUser3().code should equal(200) + And("the third request is refused by the per-minute limit: the limit is still enforced, once per request") + val refused = callV510AsUser3() + refused.code should equal(429) + val message = refused.body.extract[ErrorMessage].message + message should startWith(TooManyRequests) + message should include("per minute") + message should include(consumerId3) + } finally { + deleteLimit(consumerId3, id) + Redis.deleteKeysByPattern(s"${Constant.CALL_COUNTER_PREFIX}${consumerId3}_*") + } + } + scenario("A record with -1 in every period is unlimited, not blocked", ApiEndpoint4, VersionOfApi) { Given("A record with -1 everywhere") val id = createLimit(consumerId3, callLimitJson("-1", "-1", "-1")) From 0dd07254d8af6834b26c2b718153279cab78b226 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 8 Sep 2026 06:03:44 +0200 Subject: [PATCH 11/13] Always count call counters even if limit is -1 (unlimited) so we see activity. Ability to change dynamic entity e.g. indexing if change doesn't change structure / field names. --- .../dynamic/entity/Http4sDynamicEntity.scala | 9 +- .../entity/helper/DynamicEntityHelper.scala | 77 ++++++++++- .../dynamic/entity/query/QueryPlanner.scala | 73 ++++++++-- .../scala/code/api/util/ErrorMessages.scala | 1 + .../main/scala/code/api/util/Glossary.scala | 5 +- .../code/api/util/RateLimitingUtil.scala | 79 +++++------ .../scala/code/api/v4_0_0/Http4s400.scala | 53 +++++--- .../helper/SchemaCompatibleChangeSpec.scala | 62 +++++++++ .../dynamic/entity/query/JoinQuerySpec.scala | 52 +++++++ .../code/api/v4_0_0/DynamicEntityTest.scala | 62 +++++++++ ...ynamicEntityJoinQueryIntegrationTest.scala | 47 ++++++- .../code/api/v6_0_0/RateLimitsTest.scala | 127 +++++++++++++++++- 12 files changed, 559 insertions(+), 88 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/dynamic/entity/helper/SchemaCompatibleChangeSpec.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala index 4a827ed04a..a159903648 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala @@ -114,15 +114,20 @@ object Http4sDynamicEntity extends MdcLoggable { private def deReferenceFields(bankId: Option[String], entityName: String): Map[String, String] = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).map(_.referenceFields).getOrElse(Map.empty) + private def deUnindexedReferenceFields(bankId: Option[String], entityName: String): Map[String, String] = + DynamicEntityHelper.definitionsMap.get((bankId, entityName)).map(_.unindexedReferenceFields).getOrElse(Map.empty) + /** Resolve a join-target (child) entity's indexed + reference fields for the planner (same bank scope). */ private def childJoinInfo(bankId: Option[String])(child: String): Option[JoinTargetInfo] = - DynamicEntityHelper.definitionsMap.get((bankId, child)).map(i => JoinTargetInfo(i.indexedFields, i.referenceFields)) + DynamicEntityHelper.definitionsMap.get((bankId, child)) + .map(i => JoinTargetInfo(i.indexedFields, i.referenceFields, i.unindexedReferenceFields)) /** Parse + validate list-read query params into a QueryPlan; fail 400 (clear message) on any error. */ private def buildQueryPlan(req: Request[IO], bankId: Option[String], entityName: String, cc: Option[CallContext]): Future[QueryPlan] = { val planned = QueryParamParser.parse(queryParams(req)).flatMap { case (filters, joins, sort, page) => QueryPlanner.plan(filters, joins, sort, page, entityName, - deIndexedFields(bankId, entityName), deReferenceFields(bankId, entityName), childJoinInfo(bankId)) + deIndexedFields(bankId, entityName), deReferenceFields(bankId, entityName), childJoinInfo(bankId), + deUnindexedReferenceFields(bankId, entityName)) } planned match { case Right(plan) => Future.successful(plan) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 156f7da378..7d4a9699bb 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -145,6 +145,51 @@ object EntityAccessName { } object DynamicEntityHelper { + + /** + * DE_indexing: may this definition update be applied to an entity that already has rows? + * + * The stored rows stay valid when the entity name, the set of property names and each property's `type` + * are unchanged and `required` does not grow. Everything else — `indexed`, `index`, `example`, + * `description`, `minLength`, `maxLength`, `read_role*`, `write_role*` — may change freely; in + * particular this is what lets an operator switch indexing on for an existing, populated entity + * (the projection backfill then does the rest). Unparseable input is treated as incompatible. + */ + def isSchemaCompatibleChange(oldEntityName: String, oldMetadataJson: String, + newEntityName: String, newMetadataJson: String): Boolean = { + // The stored metadataJson is the whole definition request, i.e. `{"": {"properties": ...}}` + // (DynamicEntityCommons.apply keeps the outer object). Accept the bare inner object too. + def definitionOf(metadataJson: String, entityName: String): Option[JValue] = + scala.util.Try(parse(metadataJson)).toOption.flatMap { + case root: JObject => + (root \ entityName) match { + case inner: JObject => Some(inner) + case _ if (root \ "properties").isInstanceOf[JObject] => Some(root) + case _ => None + } + case _ => None + } + def propertyTypes(definition: JValue): Map[String, String] = + (definition \ "properties") match { + case props: JObject => props.obj.map { case JField(name, propDef) => + name -> ((propDef \ "type") match { case JString(t) => t; case _ => "" }) + }.toMap + case _ => Map.empty[String, String] + } + def requiredNames(definition: JValue): Set[String] = + (definition \ "required") match { + case JArray(items) => items.collect { case JString(n) => n }.toSet + case _ => Set.empty[String] + } + + oldEntityName == newEntityName && { + (definitionOf(oldMetadataJson, oldEntityName), definitionOf(newMetadataJson, newEntityName)) match { + case (Some(oldDef), Some(newDef)) => + propertyTypes(oldDef) == propertyTypes(newDef) && requiredNames(newDef).subsetOf(requiredNames(oldDef)) + case _ => false + } + } + } private val implementedInApiVersion = ApiVersion.v4_0_0 // (Some(BankId), EntityName, DynamicEntityInfo) @@ -927,22 +972,40 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt } /** - * Indexed `reference:` fields: fieldName -> target entity name (the part after "reference:"). - * The join planner uses this to resolve one-hop EXISTS/NOT EXISTS edges between entities; only - * declared reference fields are joinable (a plain string field holding ids is not). See - * ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + * Every `reference:` field, indexed or not: fieldName -> target entity name (the part after + * "reference:"). Only the indexed subset ([[referenceFields]]) forms a join edge; the rest + * ([[unindexedReferenceFields]]) exist so the planner can tell a developer precisely which field to + * mark `"indexed": true` instead of claiming no reference is declared at all. */ - lazy val referenceFields: Map[String, String] = (entity \ "properties") match { + lazy val allReferenceFields: Map[String, String] = (entity \ "properties") match { case props: JObject => props.obj.collect { case JField(name, propDef: JObject) - if (propDef \ "indexed") == JBool(true) && - ((propDef \ "type") match { case JString(s) => s.startsWith("reference:"); case _ => false }) => + if ((propDef \ "type") match { case JString(s) => s.startsWith("reference:"); case _ => false }) => val target = ((propDef \ "type"): @unchecked) match { case JString(s) => s.stripPrefix("reference:") } name -> target }.toMap case _ => Map.empty } + private lazy val indexedPropertyNames: Set[String] = (entity \ "properties") match { + case props: JObject => props.obj.collect { + case JField(name, propDef: JObject) if (propDef \ "indexed") == JBool(true) => name + }.toSet + case _ => Set.empty + } + + /** + * Indexed `reference:` fields: fieldName -> target entity name (the part after "reference:"). + * The join planner uses this to resolve one-hop EXISTS/NOT EXISTS edges between entities; only + * declared reference fields are joinable (a plain string field holding ids is not). See + * ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + */ + lazy val referenceFields: Map[String, String] = + allReferenceFields.filter { case (name, _) => indexedPropertyNames.contains(name) } + + /** `reference:` fields that are declared but NOT `indexed`, so they cannot be joined on (yet). */ + lazy val unindexedReferenceFields: Map[String, String] = allReferenceFields -- referenceFields.keys + /** * Human-facing documentation of the list-endpoint query grammar (filter / sort / paginate / one-hop * joins), appended to the GET-all ResourceDoc descriptions. `joinsSupported` is true on the diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala index 8d1276f13a..a4b47827ff 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala @@ -9,8 +9,13 @@ import scala.util.Try case class FieldSpec(fieldType: DynamicEntityFieldType, indexKind: String) /** What the planner needs to know about a join target (child) entity: its indexed fields (for nested - * predicate validation) and its declared reference fields (for edge inference). */ -case class JoinTargetInfo(indexedFields: Map[String, FieldSpec], referenceFields: Map[String, String]) + * predicate validation), its indexed reference fields (for edge inference), and its reference fields + * that are declared but not indexed (only used to word a precise rejection). */ +case class JoinTargetInfo( + indexedFields: Map[String, FieldSpec], + referenceFields: Map[String, String], + unindexedReferenceFields: Map[String, String] = Map.empty +) /** A contract-layer validation failure (maps to HTTP 400 at the endpoint). */ case class QueryError(message: String) @@ -38,9 +43,11 @@ object QueryPlanner { /** * Full planner including one-hop join clauses. `parentReferenceFields` are the queried entity's - * declared reference fields (for parent→child edges); `childInfoOf` resolves a join-target entity's - * indexed + reference fields (None if no such entity). Joins are resolved to a concrete link field + - * direction here, or rejected with a clear 400. + * declared and indexed reference fields (for parent→child edges); `childInfoOf` resolves a join-target + * entity's indexed + reference fields (None if no such entity); `parentUnindexedReferenceFields` are + * the queried entity's reference fields that are declared but not indexed, used only to word a + * precise rejection. Joins are resolved to a concrete link field + direction here, or rejected with + * a clear 400. */ def plan( filters: List[Filter], @@ -50,57 +57,97 @@ object QueryPlanner { parentEntityName: String, parentIndexedFields: Map[String, FieldSpec], parentReferenceFields: Map[String, String], - childInfoOf: String => Option[JoinTargetInfo] + childInfoOf: String => Option[JoinTargetInfo], + parentUnindexedReferenceFields: Map[String, String] = Map.empty ): Either[QueryError, QueryPlan] = for { _ <- firstError(filters.map(validateFilter(_, parentIndexedFields))) _ <- firstError(sort.map(validateSort(_, parentIndexedFields))) - joins <- traverse(rawJoins)(resolveJoin(_, parentEntityName, parentReferenceFields, childInfoOf)) + _ <- if (rawJoins.nonEmpty && parentIndexedFields.isEmpty) Left(parentNotIndexed(parentEntityName)) else Right(()) + joins <- traverse(rawJoins)(resolveJoin(_, parentEntityName, parentReferenceFields, parentUnindexedReferenceFields, childInfoOf)) } yield QueryPlan(filters, joins, sort, page) // ----- join resolution ----- + /** A reference field that could have been a join edge but is not `indexed`: (field, onChild). */ + private type UnindexedEdge = (String, Boolean) + private def resolveJoin( raw: RawJoin, parentEntityName: String, parentReferenceFields: Map[String, String], + parentUnindexedReferenceFields: Map[String, String], childInfoOf: String => Option[JoinTargetInfo] ): Either[QueryError, JoinClause] = childInfoOf(raw.childEntity) match { case None => Left(QueryError(s"Cannot join '${raw.childEntity}': no such Dynamic Entity.")) case Some(childInfo) => // Candidate edges: a child field referencing the parent (onChild=true), or a parent field - // referencing the child (onChild=false). Edge = a declared `reference:` field only. + // referencing the child (onChild=false). Edge = a declared AND indexed `reference:` field only. val childToParent = childInfo.referenceFields.collect { case (f, t) if t == parentEntityName => (f, true) }.toList val parentToChild = parentReferenceFields.collect { case (f, t) if t == raw.childEntity => (f, false) }.toList val candidates = childToParent ++ parentToChild + // Declared-but-unindexed references in either direction: not edges, but the reason to report. + val unindexed: List[UnindexedEdge] = + childInfo.unindexedReferenceFields.collect { case (f, t) if t == parentEntityName => (f, true) }.toList ++ + parentUnindexedReferenceFields.collect { case (f, t) if t == raw.childEntity => (f, false) }.toList for { - edge <- selectEdge(raw, candidates) + edge <- selectEdge(raw, parentEntityName, candidates, unindexed) // Nested predicate validates against the CHILD's indexed fields. _ <- firstError(raw.predicate.map(validateFilter(_, childInfo.indexedFields))) } yield JoinClause(raw.quantifier, raw.childEntity, edge._1, edge._2, raw.predicate) } - private def selectEdge(raw: RawJoin, candidates: List[(String, Boolean)]): Either[QueryError, (String, Boolean)] = + private def selectEdge( + raw: RawJoin, + parentEntityName: String, + candidates: List[(String, Boolean)], + unindexed: List[UnindexedEdge] + ): Either[QueryError, (String, Boolean)] = raw.via match { case Some(field) => candidates.filter(_._1 == field) match { case single :: Nil => Right(single) case Nil => - Left(QueryError(s"No reference field '$field' links '${raw.childEntity}' to the queried entity." + - candidateHint(raw, candidates))) + unindexed.find(_._1 == field) match { + case Some(u) => Left(unindexedEdgeError(raw, parentEntityName, List(u))) + case None => + Left(QueryError(s"No reference field '$field' links '${raw.childEntity}' to the queried entity." + + candidateHint(raw, candidates))) + } case _ => Left(QueryError(s"Ambiguous link field '$field' for join with '${raw.childEntity}'.")) } case None => candidates match { case single :: Nil => Right(single) + case Nil if unindexed.nonEmpty => Left(unindexedEdgeError(raw, parentEntityName, unindexed)) case Nil => Left(QueryError(s"Cannot join '${raw.childEntity}': no declared reference links it to the queried entity. " + - "A join edge must be a field typed 'reference:'.")) + "A join edge must be a field typed 'reference:' and declared \"indexed\": true.")) case many => Left(QueryError(s"Ambiguous join with '${raw.childEntity}': multiple reference edges " + s"(${many.map(_._1).mkString(", ")}). Specify via:.")) } } + /** The reference exists but is not indexed: say exactly which field on which entity needs `"indexed": true`. */ + private def unindexedEdgeError(raw: RawJoin, parentEntityName: String, unindexed: List[UnindexedEdge]): QueryError = + unindexed match { + case (field, onChild) :: Nil => + val (owner, target) = if (onChild) (raw.childEntity, parentEntityName) else (parentEntityName, raw.childEntity) + QueryError(s"Cannot join '${raw.childEntity}' via '$field': the field '$field' on '$owner' is typed 'reference:$target' " + + s"but is not declared \"indexed\": true. Add \"indexed\": true to that field on '$owner' " + + "(and to any field used in the nested filter) and let the index build.") + case many => + val described = many.map { case (field, onChild) => s"'$field' on '${if (onChild) raw.childEntity else parentEntityName}'" } + QueryError(s"Cannot join '${raw.childEntity}': the reference fields linking it to the queried entity " + + s"(${described.mkString(", ")}) are not declared \"indexed\": true. Add \"indexed\": true to the one you " + + "want to join on (and to any field used in the nested filter), let the index build, then specify via:.") + } + + /** Joins run on the SQL projection, which only exists for entities with at least one indexed field. */ + private def parentNotIndexed(parentEntityName: String): QueryError = + QueryError(s"Cannot join from '$parentEntityName': none of its fields are declared \"indexed\": true, so it has no " + + s"SQL projection to join on. Declare at least one field on '$parentEntityName' as \"indexed\": true and let the index build.") + private def candidateHint(raw: RawJoin, candidates: List[(String, Boolean)]): String = if (candidates.isEmpty) "" else s" Candidates: ${candidates.map(_._1).mkString(", ")}." diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index e4e2829912..650f57f670 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -81,6 +81,7 @@ object ErrorMessages { val RowLevelAccessRequiresLocalBacking = "OBP-09020: use_row_level_access is only supported for locally-backed dynamic entities. This entity is routed to an external connector (a method routing for dynamicEntityProcess exists for it), where the row-level ACL cannot be enforced. Remove the method routing or disable use_row_level_access." val RowLevelAccessNotEnabled = "OBP-09021: The row-access endpoints are only available for dynamic entities created with use_row_level_access = true." val DynamicEntityJoinRequiresProjection = "OBP-09022: obp_exists / obp_not_exists join queries require the SQL projection backend (dynamic_entity.indexing.backend=auto on a supported database). This deployment serves Dynamic Entity reads in-memory, where joins are not supported." + val DynamicEntityUpdateNotSchemaCompatible = "OBP-09023: Operation is not allowed, because this DynamicEntity already has data. The definition of a populated entity can only be changed in schema-compatible ways: the entity name, the set of property names and each property's type must stay the same, and no property may be added to 'required'. Changing indexed, index, example, description, minLength, maxLength and the read/write role settings is allowed. Delete all the data before making a structural change." // General messages (OBP-10XXX) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 233724d6b7..1c8b4bfaf5 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -271,8 +271,9 @@ object Glossary extends MdcLoggable { | |1. **Rate Limit Records**: Stored in the `RateLimiting` table with date ranges (from_date, to_date) |2. **Multiple Records**: A consumer can have multiple active rate limit records that overlap - |3. **Aggregation**: When multiple records are active, per period: a `0` in any record blocks the period; otherwise the positive values are summed; otherwise (all `-1`) the period is unlimited + |3. **Aggregation**: When multiple records are active, per period: `-1` values are ignored and the rest (`0` or positive) are summed; a sum of `0` blocks the period; nothing to sum (all `-1`) means unlimited |4. **Enforcement**: On every API request, the system checks Redis counters against the aggregated limits + |5. **Counting**: Every served request is counted in the Redis counter of every period, whether or not that period has a limit, so the call-counter endpoints show a Consumer's activity even when nothing limits it. A blocked period (sum `0`) serves nothing, so nothing is counted under it. | |### Time Periods | @@ -298,6 +299,8 @@ object Glossary extends MdcLoggable { |- `X-Rate-Limit-Remaining`: Remaining requests in current period |- `X-Rate-Limit-Reset`: Seconds until the limit resets | + |The three headers describe the shortest period that has a positive limit (per second before per minute, and so on). When no period is limited they read `-1`. + | |### HTTP Status Codes | |- **200 OK**: Request allowed, headers show current limit status diff --git a/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala b/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala index bb321da480..eb43ef4ed1 100644 --- a/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala +++ b/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala @@ -236,19 +236,6 @@ object RateLimitingUtil extends MdcLoggable { } } - /** - * Increment API call counter for a consumer after successful rate limit check. - * Called after the request passes all rate limit checks to update the counters. - * - * Counters are ALWAYS incremented regardless of limit value. This provides visibility - * into consumer activity even when rate limiting is disabled (limit = -1), which is - * useful for monitoring which apps are active and verifying the counting infrastructure. - * - * @param consumerKey The consumer ID or IP address - * @param period The time period (PER_SECOND, PER_MINUTE, etc.) - * @param limit The rate limit value (-1 means disabled, but counter still incremented) - * @return (TTL in seconds, current counter value) or (-1, -1) on Redis error - */ /** Pure Redis INCR with create-if-missing for a fully-formed key. * No gates, no key formatting — call sites pass the final key and supply their own enable flags. * Returns (ttl_seconds, current_count); (-1, -1) when Redis is unreachable. */ @@ -268,13 +255,24 @@ object RateLimitingUtil extends MdcLoggable { } } - private def incrementConsumerCounters(consumerKey: String, period: LimitCallPeriod, limit: Long): (Long, Long) = { - if (useConsumerLimits && limit > 0) { - incrementCounter(createUniqueKey(consumerKey, period), period) - } else { - (-1, -1) - } - } + /** + * Count the call in one period's counter, after the request passed every limit check. + * + * Counted whatever the period's aggregated limit is, -1 (unlimited) included. Enforcement never + * reads the counter of an unlimited period - underConsumerLimits decides -1 and 0 without touching + * Redis and only compares the counter when the limit is positive - so a counter that is always + * kept changes nothing about what is refused, and gives visibility of the consumer's activity + * through the call-counter endpoints even when no limit applies. A period whose aggregated limit + * is 0 is refused before counting, so nothing is ever served or counted under it. + * + * Only `use_consumer_limits=false` skips counting, so an instance without Redis is left alone. + * + * @param consumerKey The consumer ID (or the client IP for anonymous access) + * @param period The time period (PER_SECOND, PER_MINUTE, ...) + * @return (TTL in seconds, current counter value); (-1, -1) when not counted or Redis is unreachable + */ + private def incrementConsumerCounters(consumerKey: String, period: LimitCallPeriod): (Long, Long) = + if (useConsumerLimits) incrementCounter(createUniqueKey(consumerKey, period), period) else (-1, -1) /** * Get remaining TTL (time to live) for a rate limit counter. @@ -446,31 +444,22 @@ object RateLimitingUtil extends MdcLoggable { case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x6 == false => (fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_MONTH, rl.per_month, rl.consumer_id), 429, exceededRateLimit(rl, PER_MONTH))), userAndCallContext._2) case _ => - // All limits passed - increment counters and set rate limit headers - val incrementCounters = List ( - incrementConsumerCounters(rateLimitingKey, PER_SECOND, rl.per_second), - incrementConsumerCounters(rateLimitingKey, PER_MINUTE, rl.per_minute), - incrementConsumerCounters(rateLimitingKey, PER_HOUR, rl.per_hour), - incrementConsumerCounters(rateLimitingKey, PER_DAY, rl.per_day), - incrementConsumerCounters(rateLimitingKey, PER_WEEK, rl.per_week), - incrementConsumerCounters(rateLimitingKey, PER_MONTH, rl.per_month) + // All limits passed - count the call in every period, then set the X-Rate-Limit-* + // headers from the shortest period that HAS a limit. The counter alone cannot pick + // that period: every period is counted, unlimited ones included, so a live + // per-second counter says nothing about whether a per-second limit exists. With no + // limited period the CallContext keeps its defaults and the headers read -1. + val counted: List[(LimitCallPeriod, Long, (Long, Long))] = List( + (PER_SECOND, rl.per_second, incrementConsumerCounters(rateLimitingKey, PER_SECOND)), + (PER_MINUTE, rl.per_minute, incrementConsumerCounters(rateLimitingKey, PER_MINUTE)), + (PER_HOUR, rl.per_hour, incrementConsumerCounters(rateLimitingKey, PER_HOUR)), + (PER_DAY, rl.per_day, incrementConsumerCounters(rateLimitingKey, PER_DAY)), + (PER_WEEK, rl.per_week, incrementConsumerCounters(rateLimitingKey, PER_WEEK)), + (PER_MONTH, rl.per_month, incrementConsumerCounters(rateLimitingKey, PER_MONTH)) ) - // Set rate limit headers based on the most restrictive active period - incrementCounters match { - case first :: _ :: _ :: _ :: _ :: _ :: Nil if first._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, first, PER_SECOND)) - case _ :: second :: _ :: _ :: _ :: _ :: Nil if second._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, second, PER_MINUTE)) - case _ :: _ :: third :: _ :: _ :: _ :: Nil if third._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, third, PER_HOUR)) - case _ :: _ :: _ :: fourth :: _ :: _ :: Nil if fourth._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, fourth, PER_DAY)) - case _ :: _ :: _ :: _ :: fifth :: _ :: Nil if fifth._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, fifth, PER_WEEK)) - case _ :: _ :: _ :: _ :: _ :: sixth :: Nil if sixth._1 > 0 => - (userAndCallContext._1, setXRateLimits(rl, sixth, PER_MONTH)) - case _ => - (userAndCallContext._1, userAndCallContext._2) + counted.collectFirst { case (period, limit, ttlAndCount) if limit > 0 && ttlAndCount._1 > 0 => (period, ttlAndCount) } match { + case Some((period, ttlAndCount)) => (userAndCallContext._1, setXRateLimits(rl, ttlAndCount, period)) + case None => (userAndCallContext._1, userAndCallContext._2) } } case None => // ANONYMOUS ACCESS - no consumer credentials, use IP-based limiting @@ -487,7 +476,7 @@ object RateLimitingUtil extends MdcLoggable { case _ => // Limit not exceeded - increment counter and set headers val incrementCounters = List ( - incrementConsumerCounters(consumerId, PER_HOUR, perHourLimitAnonymous) + incrementConsumerCounters(consumerId, PER_HOUR) ) incrementCounters match { case x1 :: Nil if x1._1 > 0 => diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 28e646bcda..4e8bfd1921 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -1664,14 +1664,17 @@ object Http4s400 { private def updateDynamicEntityImpl(bankId: Option[String], dynamicEntityId: String, json: JValue, cc: CallContext): Future[JValue] = for { (entity, _) <- NewStyle.function.getDynamicEntityById(bankId, dynamicEntityId, Some(cc)) + dynamicEntity <- tryOrApiFail(cc) { + DynamicEntityCommons(json.asInstanceOf[JObject], Some(dynamicEntityId), cc.userId, bankId) + } (box, _) <- NewStyle.function.invokeDynamicConnector( GET_ALL, entity.entityName, None, None, entity.bankId, None, None, false, Some(cc)) resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], entity.entityName) - _ <- code.util.Helper.booleanToFuture(DynamicEntityOperationNotAllowed, cc = Some(cc)) { - resultList.arr.isEmpty - } - dynamicEntity <- tryOrApiFail(cc) { - DynamicEntityCommons(json.asInstanceOf[JObject], Some(dynamicEntityId), cc.userId, bankId) + // A populated entity may still take a schema-compatible update (e.g. switching `indexed` on so + // DE_indexing can backfill it); a structural change still requires the data to be deleted first. + _ <- code.util.Helper.booleanToFuture(DynamicEntityUpdateNotSchemaCompatible, cc = Some(cc)) { + resultList.arr.isEmpty || code.api.dynamic.entity.helper.DynamicEntityHelper.isSchemaCompatibleChange( + entity.entityName, entity.metadataJson, dynamicEntity.entityName, dynamicEntity.metadataJson) } Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) } yield { @@ -1782,11 +1785,17 @@ object Http4s400 { "/management/system-dynamic-entities/DYNAMIC_ENTITY_ID", "Update System Level Dynamic Entity", s"""Update a system level DynamicEntity. + | + |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of + |property names and each property's `type` must stay the same, and `required` may not grow. Changing + |`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is + |allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change + |returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted. | |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canUpdateSystemDynamicEntity)), http4sPartialFunction = Some(updateSystemDynamicEntity)) @@ -1811,11 +1820,17 @@ object Http4s400 { "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID", "Update Bank Level Dynamic Entity", s"""Update a Bank Level DynamicEntity. + | + |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of + |property names and each property's `type` must stay the same, and `required` may not grow. Changing + |`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is + |allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change + |returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted. | |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canUpdateBankLevelDynamicEntity)), http4sPartialFunction = Some(updateBankLevelDynamicEntity)) @@ -1901,19 +1916,21 @@ object Http4s400 { myEntity <- NewStyle.function.tryons(InvalidMyDynamicEntityUser, 400, Some(cc)) { entityOption.get } - (box, _) <- NewStyle.function.invokeDynamicConnector( - GET_ALL, myEntity.entityName, None, myEntity.dynamicEntityId, - myEntity.bankId, None, Some(myEntity.userId), false, Some(cc)) - resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], myEntity.entityName) - _ <- code.util.Helper.booleanToFuture(DynamicEntityOperationNotAllowed, cc = Some(cc)) { - resultList.arr.isEmpty - } jsonObj <- NewStyle.function.tryons(InvalidJsonFormat, 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(rawBody).asInstanceOf[JObject] } dynamicEntity <- tryOrApiFail(cc) { DynamicEntityCommons(jsonObj, Some(dynamicEntityId), user.userId, myEntity.bankId) } + (box, _) <- NewStyle.function.invokeDynamicConnector( + GET_ALL, myEntity.entityName, None, myEntity.dynamicEntityId, + myEntity.bankId, None, Some(myEntity.userId), false, Some(cc)) + resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], myEntity.entityName) + // Same rule as updateDynamicEntityImpl: populated entities accept schema-compatible updates only. + _ <- code.util.Helper.booleanToFuture(DynamicEntityUpdateNotSchemaCompatible, cc = Some(cc)) { + resultList.arr.isEmpty || code.api.dynamic.entity.helper.DynamicEntityHelper.isSchemaCompatibleChange( + myEntity.entityName, myEntity.metadataJson, dynamicEntity.entityName, dynamicEntity.metadataJson) + } Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) } yield { val commonsData: DynamicEntityCommons = result @@ -1927,11 +1944,17 @@ object Http4s400 { "/my/dynamic-entities/DYNAMIC_ENTITY_ID", "Update My Dynamic Entity", s"""Update my DynamicEntity specified by DYNAMIC_ENTITY_ID. + | + |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of + |property names and each property's `type` must stay the same, and `required` may not grow. Changing + |`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is + |allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change + |returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted. | |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(AuthenticatedUserIsRequired, InvalidMyDynamicEntityUser, InvalidJsonFormat, UnknownError), + List(AuthenticatedUserIsRequired, InvalidMyDynamicEntityUser, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), None, http4sPartialFunction = Some(updateMyDynamicEntity)) diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/helper/SchemaCompatibleChangeSpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/helper/SchemaCompatibleChangeSpec.scala new file mode 100644 index 0000000000..f9880588f1 --- /dev/null +++ b/obp-api/src/test/scala/code/api/dynamic/entity/helper/SchemaCompatibleChangeSpec.scala @@ -0,0 +1,62 @@ +package code.api.dynamic.entity.helper + +import org.scalatest.{FlatSpec, Matchers} + +/** + * Pure unit tests for [[DynamicEntityHelper.isSchemaCompatibleChange]]: the rule that decides whether a + * definition update may be applied to a Dynamic Entity that already has rows. No server / DB. + * + * metadataJson is stored as the whole definition request, `{"FooBar": {...}}`; the bare inner object is + * accepted too. + */ +class SchemaCompatibleChangeSpec extends FlatSpec with Matchers { + + private def wrap(inner: String, name: String = "FooBar") = s"""{"$name":$inner}""" + + private val baseInner = + """{"description":"d","required":["name"],"properties":{ + | "name":{"type":"string","example":"x","maxLength":20}, + | "number":{"type":"integer","example":1}, + | "site_ref":{"type":"reference:Site","example":"abc"}}}""".stripMargin + private val base = wrap(baseInner) + + private def ok(newInner: String, newName: String = "FooBar") = + DynamicEntityHelper.isSchemaCompatibleChange("FooBar", base, newName, wrap(newInner, newName)) + + "isSchemaCompatibleChange" should "accept the identical definition, in outer or bare form" in { + ok(baseInner) shouldBe true + DynamicEntityHelper.isSchemaCompatibleChange("FooBar", base, "FooBar", baseInner) shouldBe true + } + + it should "accept adding indexed / index / description / example / min-max length / role settings" in { + ok("""{"description":"changed","required":["name"],"properties":{ + | "name":{"type":"string","example":"y","maxLength":40,"minLength":1,"indexed":true,"description":"n","read_role_required":true}, + | "number":{"type":"integer","example":2,"indexed":true,"index":"scalar","write_role":"CanX"}, + | "site_ref":{"type":"reference:Site","example":"def","indexed":true}}}""".stripMargin) shouldBe true + } + + it should "accept shrinking required" in { + ok("""{"required":[],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"integer","example":1},"site_ref":{"type":"reference:Site","example":"a"}}}""") shouldBe true + } + + it should "reject growing required" in { + ok("""{"required":["name","number"],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"integer","example":1},"site_ref":{"type":"reference:Site","example":"a"}}}""") shouldBe false + } + + it should "reject a changed property type (including a changed reference target)" in { + ok("""{"required":["name"],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"string","example":"1"},"site_ref":{"type":"reference:Site","example":"a"}}}""") shouldBe false + ok("""{"required":["name"],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"integer","example":1},"site_ref":{"type":"reference:Plot","example":"a"}}}""") shouldBe false + } + + it should "reject an added or removed property" in { + ok("""{"required":["name"],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"integer","example":1},"site_ref":{"type":"reference:Site","example":"a"},"extra":{"type":"string","example":"e"}}}""") shouldBe false + ok("""{"required":["name"],"properties":{"name":{"type":"string","example":"x"},"number":{"type":"integer","example":1}}}""") shouldBe false + } + + it should "reject a renamed entity and unparseable input" in { + ok(baseInner, newName = "FooBaz") shouldBe false + DynamicEntityHelper.isSchemaCompatibleChange("FooBar", base, "FooBar", "not json") shouldBe false + DynamicEntityHelper.isSchemaCompatibleChange("FooBar", "not json", "FooBar", base) shouldBe false + DynamicEntityHelper.isSchemaCompatibleChange("FooBar", base, "FooBar", """{"Other":{"properties":{}}}""") shouldBe false + } +} diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala index 7730572ea6..c55a2d362c 100644 --- a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala +++ b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala @@ -106,6 +106,58 @@ class JoinQuerySpec extends FlatSpec with Matchers { planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), unrelated).isLeft shouldBe true } + // ----- planner: precise rejections when the reference exists but is not indexed ----- + + // child Contract whose partner_id is typed reference:Partner but NOT declared indexed + private val contractUnindexedEdge = JoinTargetInfo( + indexedFields = Map("active" -> FieldSpec(DynamicEntityFieldType.boolean, "scalar")), + referenceFields = Map.empty, + unindexedReferenceFields = Map("partner_id" -> "Partner")) + + it should "name the unindexed child reference field instead of claiming no reference is declared" in { + val Left(err) = planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), contractUnindexedEdge) + err.message should include ("via 'partner_id'") + err.message should include ("'partner_id' on 'Contract' is typed 'reference:Partner'") + err.message should include ("not declared \"indexed\": true") + err.message should not include ("no declared reference") + } + + it should "name the unindexed child reference field when via: points at it" in { + val Left(err) = planJoin(RawJoin(Quantifier.Exists, "Contract", Some("partner_id"), Nil), contractUnindexedEdge) + err.message should include ("'partner_id' on 'Contract'") + err.message should include ("not declared \"indexed\": true") + } + + it should "name the unindexed parent reference field for a parent -> child edge" in { + val childNoEdges = JoinTargetInfo(Map("active" -> FieldSpec(DynamicEntityFieldType.boolean, "scalar")), Map.empty) + val Left(err) = QueryPlanner.plan(Nil, List(RawJoin(Quantifier.NotExists, "Contract", None, Nil)), Nil, Page.empty, + "Partner", partnerIndexed, Map.empty, childInfoOf(Map("Contract" -> childNoEdges)), + parentUnindexedReferenceFields = Map("favourite_contract" -> "Contract")) + err.message should include ("'favourite_contract' on 'Partner' is typed 'reference:Contract'") + err.message should include ("not declared \"indexed\": true") + } + + it should "list every unindexed reference field when more than one could be the edge" in { + val twoUnindexed = contractUnindexedEdge.copy(unindexedReferenceFields = Map("buyer_id" -> "Partner", "seller_id" -> "Partner")) + val Left(err) = planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), twoUnindexed) + err.message should include ("'buyer_id' on 'Contract'") + err.message should include ("'seller_id' on 'Contract'") + err.message should include ("via:") + } + + it should "reject a join from a parent that has no indexed fields at all, saying so" in { + val Left(err) = QueryPlanner.plan(Nil, List(RawJoin(Quantifier.Exists, "Contract", None, Nil)), Nil, Page.empty, + "Partner", Map.empty, Map.empty, childInfoOf(Map("Contract" -> contractSingleEdge))) + err.message should include ("Cannot join from 'Partner'") + err.message should include ("no SQL projection") + } + + it should "keep the generic message when no reference of any kind links the entities" in { + val unrelated = JoinTargetInfo(Map("active" -> FieldSpec(DynamicEntityFieldType.boolean, "scalar")), Map.empty) + val Left(err) = planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), unrelated) + err.message should include ("no declared reference") + } + it should "reject a join onto a non-existent entity" in { QueryPlanner.plan(Nil, List(RawJoin(Quantifier.Exists, "Nope", None, Nil)), Nil, Page.empty, "Partner", partnerIndexed, Map.empty, _ => None).isLeft shouldBe true diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala index 5ff9137de0..c2bea5273d 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala @@ -1635,6 +1635,68 @@ class DynamicEntityTest extends V400ServerSetup { } + feature("Update a populated Dynamic Entity: schema-compatible changes only") { + scenario("indexed:true can be switched on with data present; structural changes are still refused", ApiEndpoint1, ApiEndpoint4, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateSystemLevelDynamicEntity.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanUpdateSystemLevelDynamicEntity.toString) + + When("we create the FooBar entity and insert one record") + val response = makePostRequest((v4_0_0_Request / "management" / "system-dynamic-entities").POST <@(user1), write(rightEntity)) + response.code should equal(201) + val dynamicEntityId = (response.body \ "dynamicEntityId").asInstanceOf[JString].s + val foobarObject = parse("""{"name":"James Brown","number":698761728}""") + makePostRequest((dynamicEntity_Request / "FooBar").POST <@(user1), write(foobarObject)).code should equal(201) + + val updateRequest = (v4_0_0_Request / "management" / "system-dynamic-entities" / dynamicEntityId).PUT <@(user1) + + Then("switching indexed on (and touching description / example / maxLength) is accepted") + val indexOnly = rightEntity + .transformField { case JField("name", JObject(fields)) => + JField("name", JObject(fields.map { + case JField("maxLength", _) => JField("maxLength", JInt(40)) + case JField("description", _) => JField("description", JString("now indexed")) + case other => other + } :+ JField("indexed", JBool(true)))) + } + val okResponse = makePutRequest(updateRequest, write(indexOnly)) + okResponse.code should equal(200) + (okResponse.body \ "FooBar" \ "properties" \ "name" \ "indexed") should equal(JBool(true)) + + And("the record is still there") + val getAll = makeGetRequest((dynamicEntity_Request / "FooBar").GET <@(user1)) + getAll.code should equal(200) + (getAll.body \ "foo_bar_list").asInstanceOf[JArray].arr.size should equal(1) + + Then("changing a property's type is refused with " + DynamicEntityUpdateNotSchemaCompatible) + val typeChanged = rightEntity.transformField { case JField("number", JObject(fields)) => + JField("number", JObject(fields.map { + case JField("type", _) => JField("type", JString("string")) + case JField("example", _) => JField("example", JString("69876172")) + case other => other + })) + } + val typeResponse = makePutRequest(updateRequest, write(typeChanged)) + typeResponse.code should equal(400) + typeResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible) + + Then("adding a property is refused") + val propertyAdded = rightEntity.transformField { case JField("properties", JObject(fields)) => + JField("properties", JObject(fields :+ JField("colour", JObject(List(JField("type", JString("string")), JField("example", JString("red"))))))) + } + val addResponse = makePutRequest(updateRequest, write(propertyAdded)) + addResponse.code should equal(400) + addResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible) + + Then("making an existing optional property required is refused") + val requiredGrown = rightEntity.transformField { case JField("required", JArray(items)) => + JField("required", JArray(items :+ JString("number"))) + } + val requiredResponse = makePutRequest(updateRequest, write(requiredGrown)) + requiredResponse.code should equal(400) + requiredResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible) + } + } + feature("Test personal CRUD Records.") { scenario("User1 Create System Foobar, user1 and user2 both CRUD their own myFooBars. ", ApiEndpoint1, VersionOfApi) { Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateSystemLevelDynamicEntity.toString) diff --git a/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala index ad6d075945..95e773f0fa 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala @@ -36,12 +36,14 @@ class DynamicEntityJoinQueryIntegrationTest extends V600ServerSetup { private val Deal = s"Deal$sfx" private def idField(entity: String): String = StringHelpers.snakify(entity) + "_id" - private def createDef(entity: String, propsJson: String, rowLevel: Boolean = false): Unit = { + /** Create (dynamicEntityId = None) or update (Some(id)) a definition; returns its dynamicEntityId. */ + private def createDef(entity: String, propsJson: String, rowLevel: Boolean = false, existingId: Option[String] = None): String = { val metadata = s"""{"$entity":{"properties":$propsJson}}""" DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate( - DynamicEntityCommons(entity, metadata, None, owner, None, hasPersonalEntity = false, + DynamicEntityCommons(entity, metadata, existingId, owner, None, hasPersonalEntity = false, hasCommunityAccess = true, useRowLevelAccess = rowLevel) ).openOrThrowException(s"failed to create definition for $entity") + .dynamicEntityId.getOrElse(throw new IllegalStateException(s"no dynamicEntityId for $entity")) } /** Save a record (explicit id so references are controllable). Returns the record's id (= DynamicDataId). */ @@ -118,5 +120,46 @@ class DynamicEntityJoinQueryIntegrationTest extends V600ServerSetup { // userB has no grants -> no readable deals -> no partner matches. queryPartnerIds(dealExists, userB) shouldBe Set.empty[String] } + + scenario("indexing is switched on for entities that already have rows, then a join works (backfill)") { + if (!APIUtil.getPropsAsBoolValue("test.projection.postgres", false) || IndexingCapabilities.vendor != IndexingCapabilities.Postgres) + cancel("Postgres projection integration tests disabled (set test.projection.postgres=true with a Postgres db.url).") + + val Site = s"Site$sfx" + val Visit = s"Visit$sfx" + + // --- definitions WITHOUT indexed: the shape OGCR had before wanting joins --- + val siteId = createDef(Site, s"""{"${idField(Site)}":{"type":"string"},"region":{"type":"string"}}""") + val visitId = createDef(Visit, s"""{"${idField(Visit)}":{"type":"string"},"site_ref":{"type":"reference:$Site"},"done":{"type":"boolean"}}""") + + // --- rows exist before any field is indexed --- + val s1 = saveRec(Site, "region" -> JString("north")) + val s2 = saveRec(Site, "region" -> JString("south")) + saveRec(Visit, "site_ref" -> JString(s1), "done" -> JBool(true)) + saveRec(Visit, "site_ref" -> JString(s2), "done" -> JBool(false)) + + // --- the schema-compatible update: same names and types, only `indexed` added --- + val siteUnindexed = DynamicEntityProvider.connectorMethodProvider.vend.getById(None, siteId).openOrThrowException("site def").metadataJson + val visitUnindexed = DynamicEntityProvider.connectorMethodProvider.vend.getById(None, visitId).openOrThrowException("visit def").metadataJson + val siteIndexedProps = s"""{"${idField(Site)}":{"type":"string"},"region":{"type":"string","indexed":true}}""" + val visitIndexedProps = s"""{"${idField(Visit)}":{"type":"string"},"site_ref":{"type":"reference:$Site","indexed":true},"done":{"type":"boolean","indexed":true}}""" + code.api.dynamic.entity.helper.DynamicEntityHelper.isSchemaCompatibleChange( + Site, siteUnindexed, Site, s"""{"$Site":{"properties":$siteIndexedProps}}""") shouldBe true + code.api.dynamic.entity.helper.DynamicEntityHelper.isSchemaCompatibleChange( + Visit, visitUnindexed, Visit, s"""{"$Visit":{"properties":$visitIndexedProps}}""") shouldBe true + createDef(Site, siteIndexedProps, existingId = Some(siteId)) + createDef(Visit, visitIndexedProps, existingId = Some(visitId)) + + // --- provision after the update: the backfill must pick up the pre-existing rows --- + List(Site, Visit).foreach(e => run(ProjectionProvisioner.ensureProvisioned(None, e))) + + def siteIds(plan: QueryPlan): Set[String] = + PostgresProjectionBackend.query(Site, None, Some(owner), isPersonalEntity = false, plan) + .map(_.flatMap(o => (o \ idField(Site)) match { case JString(x) => Some(x); case _ => None }).toSet) + .unsafeRunSync() + val doneTrue = List(Filter("done", FilterOp.Eq, List("true"))) + siteIds(QueryPlan(Nil, List(JoinClause(Quantifier.Exists, Visit, "site_ref", onChild = true, doneTrue)), Nil, Page.empty)) shouldBe Set(s1) + siteIds(QueryPlan(Nil, List(JoinClause(Quantifier.NotExists, Visit, "site_ref", onChild = true, doneTrue)), Nil, Page.empty)) shouldBe Set(s2) + } } } diff --git a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala index 17b89caaf4..918cfc30c2 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala @@ -27,8 +27,9 @@ package code.api.v6_0_0 import org.json4s._ import code.api.util.APIUtil.OAuth._ -import code.api.Constant +import code.api.{Constant, JedisMethod} import code.api.cache.Redis +import code.setup.OBPReq import code.api.util.ApiRole.{CanCreateRateLimits, CanDeleteRateLimits, CanGetRateLimits} import code.api.util.ErrorMessages.{UserHasMissingRoles, AuthenticatedUserIsRequired, TooManyRequests} import code.api.v6_0_0.Http4s600.Implementations6_0_0 @@ -398,7 +399,7 @@ class RateLimitsTest extends V600ServerSetup { Given("A record limiting the consumer to 2 calls per minute, unlimited otherwise") // Earlier scenarios in this class called the API as user3 within the same minute, and // counters are incremented even under an unlimited record: start this window from zero. - Redis.deleteKeysByPattern(s"${Constant.CALL_COUNTER_PREFIX}${consumerId3}_*") + resetCallCounters(consumerId3) val id = createLimit(consumerId3, callLimitJson("-1", "2", "-1")) try { def callV510AsUser3() = makeGetRequest((v5_1_0_Request / "users" / "current").GET <@ (user3)) @@ -418,7 +419,7 @@ class RateLimitsTest extends V600ServerSetup { message should include(consumerId3) } finally { deleteLimit(consumerId3, id) - Redis.deleteKeysByPattern(s"${Constant.CALL_COUNTER_PREFIX}${consumerId3}_*") + resetCallCounters(consumerId3) } } @@ -440,4 +441,124 @@ class RateLimitsTest extends V600ServerSetup { } } } + + // --------------------------------------------------------------------------------------------- + // Version fallthrough. A request arriving at /obp/vX/... is offered to every version's route group + // in turn (Http4sApp: v7.0.0, v6.0.0, v5.1.0, v5.0.0, Berlin Group, UK, v4.0.0, ... v1.2.1), and a + // group without an endpoint for it may hand it to an older version through its bridge + // (v7.0.0 -> v6.0.0 -> v5.1.0 -> ... -> v1.2.1). However many groups pass it on and however many + // bridges it crosses, ONE request must cost ONE rate-limit unit. It used to cost one per hop. + // + // Counters are read straight from Redis. RateLimitingUtil counts every served request in every + // period, whether or not a limit applies, so they are an exact per-request count that does not + // depend on timing. + // --------------------------------------------------------------------------------------------- + + def callCounterKey(consumerId: String, period: String): String = s"${Constant.CALL_COUNTER_PREFIX}${consumerId}_$period" + + def callCounter(consumerId: String, period: String): Long = + Redis.use(JedisMethod.GET, callCounterKey(consumerId, period)).map(_.toLong).getOrElse(0L) + + def resetCallCounters(consumerId: String): Unit = + Redis.deleteKeysByPattern(s"${Constant.CALL_COUNTER_PREFIX}${consumerId}_*") + + /** + * @param requestVersion the version prefix the request arrives at + * @param path path segments after the version + * @param servedBy which version serves it and how it gets there (documentation for the scenario title) + * @param versionServed expected `X-OBP-Version-Served` response header. Only the v7.0.0 -> v6.0.0, + * v6.0.0 -> v5.1.0 and v5.1.0 -> v5.0.0 bridges stamp it, and it names the + * first bridge crossed, not necessarily the version that finally serves. + */ + case class FallthroughCase(requestVersion: String, path: List[String], servedBy: String, versionServed: Option[String]) + + val fallthroughCases: List[FallthroughCase] = List( + FallthroughCase("v7.0.0", List("users", "current"), "v7.0.0 itself, the first group in the chain", None), + FallthroughCase("v6.0.0", List("users", "current"), "v6.0.0 itself, after the v7.0.0 group passed it on", None), + FallthroughCase("v7.0.0", List("banks", "testBank0"), "v6.0.0 through the v7.0.0 -> v6.0.0 bridge", Some("v6.0.0")), + FallthroughCase("v7.0.0", List("banks"), "v6.0.0 through the v7.0.0 -> v6.0.0 bridge", Some("v6.0.0")), + FallthroughCase("v5.1.0", List("users", "current"), "v3.0.0 through the v5.1.0 -> v5.0.0 -> v4.0.0 -> v3.1.0 -> v3.0.0 bridges, after the v7.0.0 and v6.0.0 groups passed it on", Some("v5.0.0")), + FallthroughCase("v4.0.0", List("users", "current"), "v3.0.0 through the v4.0.0 -> v3.1.0 -> v3.0.0 bridges, after nine groups (v7.0.0 down to Berlin Group and UK) passed it on", None), + FallthroughCase("v3.0.0", List("users", "current"), "v3.0.0 itself, after eleven groups passed it on", None), + FallthroughCase("v2.2.0", List("users", "current"), "v2.0.0 through the v2.2.0 -> v2.1.0 -> v2.0.0 bridges", None), + FallthroughCase("v2.0.0", List("banks"), "v1.2.1 through the v2.0.0 -> v1.4.0 -> v1.3.0 -> v1.2.1 bridges", None), + FallthroughCase("v1.2.1", List("banks"), "v1.2.1 itself, the last OBP group in the chain", None) + ) + + def requestFor(c: FallthroughCase): OBPReq = + c.path.foldLeft(baseRequest / "obp" / c.requestVersion)(_ / _).GET <@ (user3) + + feature("Rate limiting counts one unit per request, whichever version prefix it arrives at and however many hops it crosses") { + fallthroughCases.foreach { c => + scenario(s"GET /obp/${c.requestVersion}/${c.path.mkString("/")} is served by ${c.servedBy}, and costs one unit", ApiEndpoint4, VersionOfApi) { + Given("no rate limit record for the consumer, and its call counters at zero") + activeLimitsNow(consumerId3).considered_rate_limit_ids shouldBe empty // made by user1, so not counted for consumer3 + resetCallCounters(consumerId3) + When("the consumer makes the request once") + val first = makeGetRequest(requestFor(c)) + Then("it is served") + first.code should equal(200) + And(s"X-OBP-Version-Served is ${c.versionServed.getOrElse("absent")}") + first.headers.flatMap(h => Option(h.get("X-OBP-Version-Served"))) should equal(c.versionServed) + And("the X-Rate-Limit headers read -1: no period is limited") + first.headers.flatMap(h => Option(h.get("X-Rate-Limit-Limit"))) should equal(Some("-1")) + And("the per-minute and per-hour counters still read 1: activity is counted even when nothing limits it, one unit per request whatever the hop count") + callCounter(consumerId3, "PER_MINUTE") should equal(1L) + callCounter(consumerId3, "PER_HOUR") should equal(1L) + And("a second request makes them 2") + makeGetRequest(requestFor(c)).code should equal(200) + callCounter(consumerId3, "PER_MINUTE") should equal(2L) + callCounter(consumerId3, "PER_HOUR") should equal(2L) + resetCallCounters(consumerId3) + } + } + + scenario("X-Rate-Limit headers describe the shortest LIMITED period, not merely the shortest counted one", ApiEndpoint4, VersionOfApi) { + Given("A record with per second unlimited, 100 per minute and 1000 per hour, and counters at zero") + // Every period is counted, so the per-second counter is live too; the headers must skip it + // because it has no limit, and describe the per-minute limit. + val id = createLimit(consumerId3, callLimitJson("-1", "100", "1000")) + try { + resetCallCounters(consumerId3) + When("the consumer makes a request") + val first = callAsUser3() + Then("it is served with the per-minute limit and remaining calls in the headers") + first.code should equal(200) + first.headers.flatMap(h => Option(h.get("X-Rate-Limit-Limit"))) should equal(Some("100")) + first.headers.flatMap(h => Option(h.get("X-Rate-Limit-Remaining"))) should equal(Some("99")) + And("the per-second counter was still counted") + callCounter(consumerId3, "PER_SECOND") should be >= 1L + And("a second request reports one fewer remaining") + callAsUser3().headers.flatMap(h => Option(h.get("X-Rate-Limit-Remaining"))) should equal(Some("98")) + } finally { + deleteLimit(consumerId3, id) + resetCallCounters(consumerId3) + } + } + + // v4.0.0/users/current: nine groups pass it on, then two bridges, then v3.0.0 serves it. It is the + // deepest NEW-style target: the old-style versions (v2.0.0 and below) report a refused call as + // 400 rather than 429 (ResourceDocMiddleware.authenticate keeps Lift's old-style status codes). + scenario("A per-minute limit of 2 is enforced once per request deep in the chain (v4.0.0/users/current, served by v3.0.0)", ApiEndpoint4, VersionOfApi) { + Given("A record limiting the consumer to 2 calls per minute, unlimited otherwise, and counters at zero") + resetCallCounters(consumerId3) + val id = createLimit(consumerId3, callLimitJson("-1", "2", "-1")) + try { + val deepest = fallthroughCases.find(c => c.requestVersion == "v4.0.0" && c.path == List("users", "current")).get + When("the consumer makes three requests") + Then("the first two are served and the third is refused with 429 for the per-minute limit") + makeGetRequest(requestFor(deepest)).code should equal(200) + makeGetRequest(requestFor(deepest)).code should equal(200) + val refused = makeGetRequest(requestFor(deepest)) + refused.code should equal(429) + val message = refused.body.extract[ErrorMessage].message + message should startWith(TooManyRequests) + message should include("per minute") + message should include(consumerId3) + } finally { + deleteLimit(consumerId3, id) + resetCallCounters(consumerId3) + } + } + } } From 0e7a472922c54a91dddfffd1f7d45360b794515e Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 8 Sep 2026 12:26:50 +0200 Subject: [PATCH 12/13] rate limit / call context headers --- .../main/scala/code/api/util/APIUtil.scala | 9 ++- .../util/http4s/ErrorResponseConverter.scala | 55 ++++++++----------- .../code/api/util/http4s/Http4sSupport.scala | 40 +++++++++----- .../util/http4s/ResourceDocMiddleware.scala | 12 ++-- .../scala/code/api/v2_1_0/Http4s210.scala | 2 +- .../scala/code/api/v2_2_0/Http4s220.scala | 6 +- .../scala/code/api/v3_0_0/Http4s300.scala | 4 +- .../code/api/v6_0_0/RateLimitsTest.scala | 3 + 8 files changed, 71 insertions(+), 60 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 64058b0ee9..8ffde2526c 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -514,7 +514,14 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ } } - private def getHeadersNewStyle(cc: Option[CallContextLight]) = { + /** + * Response headers derived from the CallContext: GatewayLogin, ASPSP-SCA-Approach (Berlin Group + * consents), X-Rate-Limit-Limit / -Remaining / -Reset, the pagination Range header, request + * headers mirrored back (`mirror_request_headers_to_response`) and echoed back + * (`echo_request_headers`). Lift's futureToResponse added these to every response; on http4s + * EndpointHelpers (success) and ErrorResponseConverter (errors) do. + */ + def getHeadersNewStyle(cc: Option[CallContextLight]): CustomResponseHeaders = { CustomResponseHeaders( getGatewayLoginHeader(cc).list ::: getRequestHeadersBerlinGroup(cc).list ::: diff --git a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala index 8d1b870795..1ebbf7b797 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala @@ -9,7 +9,7 @@ import code.api.berlin.group.v1_3.JSONFactory_BERLIN_GROUP_1_3.{ErrorMessageBG, import code.api.util.APIUtil.JsonResponseExtractor import code.api.util.BerlinGroupError import code.api.util.ErrorMessages._ -import code.api.util.CallContext +import code.api.util.{CallContext, CallContextLight} import net.liftweb.common.{Failure => LiftFailure} import com.openbankproject.commons.util.JsonAliases.parse import org.json4s.Extraction @@ -41,6 +41,20 @@ object ErrorResponseConverter { implicit val formats: Formats = CustomJsonFormats.formats private val jsonContentType: `Content-Type` = `Content-Type`(MediaType.application.json) + /** + * Correlation-Id plus the CallContext-derived headers of APIUtil.getHeadersNewStyle + * (X-Rate-Limit-*, GatewayLogin, mirrored and echoed request headers, ...). The failure's own + * CallContextLight is preferred when it carries one: a 429 from RateLimitingUtil.underCallLimits + * stamps the exhausted limit and its reset time there, while the CallContext the middleware still + * holds predates the rate-limit check. + */ + private def withResponseHeaders(response: Response[IO], callContext: CallContext, ccl: Option[CallContextLight]): Response[IO] = { + val withCorrelationId = response.putHeaders(Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) + code.api.util.APIUtil.getHeadersNewStyle(Some(ccl.getOrElse(callContext.toLight))).list.foldLeft(withCorrelationId) { + case (r, (name, value)) => r.putHeaders(Header.Raw(CIString(name), value)) + } + } + private val obpErrorCodePrefix = "^OBP-\\d{5}: ".r private def tryExtractApiFailureFromExceptionMessage(error: Throwable): Option[APIFailureNewStyle] = { @@ -50,7 +64,7 @@ object ErrorResponseConverter { val jv = parse(msg) val failCode = (jv \ "failCode").extract[Int] val failMsg = (jv \ "failMsg").extract[String] - Some(APIFailureNewStyle(failMsg, failCode)) + Some(APIFailureNewStyle(failMsg, failCode, (jv \ "ccl").extractOpt[CallContextLight])) } catch { case _: Throwable => None } @@ -119,12 +133,7 @@ object ErrorResponseConverter { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(code, message, callContext) else toJsonString(OBPErrorResponse(code, message)) val status = org.http4s.Status.fromInt(code).getOrElse(org.http4s.Status.BadRequest) - IO.pure( - Response[IO](status) - .withEntity(body) - .withContentType(jsonContentType) - .putHeaders(org.http4s.Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - ) + IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, None)) } /** Old-style versions keep raw 400 codes — they never promote to 403/401/etc. @@ -153,12 +162,7 @@ object ErrorResponseConverter { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(resolvedCode, failure.failMsg, callContext) else toJsonString(OBPErrorResponse(resolvedCode, failure.failMsg)) val status = org.http4s.Status.fromInt(resolvedCode).getOrElse(org.http4s.Status.BadRequest) - IO.pure( - Response[IO](status) - .withEntity(body) - .withContentType(jsonContentType) - .putHeaders(org.http4s.Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - ) + IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, failure.ccl)) } /** @@ -168,12 +172,7 @@ object ErrorResponseConverter { def boxFailureToResponse(failure: LiftFailure, callContext: CallContext): IO[Response[IO]] = { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(400, failure.msg, callContext) else toJsonString(OBPErrorResponse(400, failure.msg)) - IO.pure( - Response[IO](org.http4s.Status.BadRequest) - .withEntity(body) - .withContentType(jsonContentType) - .putHeaders(org.http4s.Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - ) + IO.pure(withResponseHeaders(Response[IO](org.http4s.Status.BadRequest).withEntity(body).withContentType(jsonContentType), callContext, None)) } /** @@ -185,26 +184,16 @@ object ErrorResponseConverter { val message = s"$UnknownError: ${e.getMessage}" val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(500, message, callContext) else toJsonString(OBPErrorResponse(500, message)) - IO.pure( - Response[IO](org.http4s.Status.InternalServerError) - .withEntity(body) - .withContentType(jsonContentType) - .putHeaders(org.http4s.Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - ) + IO.pure(withResponseHeaders(Response[IO](org.http4s.Status.InternalServerError).withEntity(body).withContentType(jsonContentType), callContext, None)) } /** * Create error response with specific status code and message. */ - def createErrorResponse(statusCode: Int, message: String, callContext: CallContext): IO[Response[IO]] = { + def createErrorResponse(statusCode: Int, message: String, callContext: CallContext, ccl: Option[CallContextLight] = None): IO[Response[IO]] = { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(statusCode, message, callContext) else toJsonString(OBPErrorResponse(statusCode, message)) val status = org.http4s.Status.fromInt(statusCode).getOrElse(org.http4s.Status.BadRequest) - IO.pure( - Response[IO](status) - .withEntity(body) - .withContentType(jsonContentType) - .putHeaders(org.http4s.Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - ) + IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, ccl)) } } diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index dddec1463d..f3483fd307 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -142,9 +142,21 @@ object Http4sRequestAttributes { // clients (and the strict application/json check in the frontend) see the real type. private val jsonContentType = `Content-Type`(MediaType.application.json) - private def toJsonOk[A](result: A)(implicit formats: Formats): IO[Response[IO]] = { + /** + * Add the CallContext-derived headers (APIUtil.getHeadersNewStyle): X-Rate-Limit-Limit / + * -Remaining / -Reset, GatewayLogin, ASPSP-SCA-Approach, pagination Range, mirrored and + * echoed request headers. RateLimitingUtil.underCallLimits stamps the rate-limit values on + * the CallContext during ResourceDocMiddleware.authenticate; this is where they reach the + * client. Lift's futureToResponse did the same for every response. + */ + def withCallContextHeaders(response: Response[IO])(implicit cc: CallContext): Response[IO] = + code.api.util.APIUtil.getHeadersNewStyle(Some(cc.toLight)).list.foldLeft(response) { + case (r, (name, value)) => r.putHeaders(Header.Raw(CIString(name), value)) + } + + private def toJsonOk[A](result: A)(implicit formats: Formats, cc: CallContext): IO[Response[IO]] = { val jsonString = prettyRender(Extraction.decompose(result)) - Ok(jsonString, jsonContentType) + Ok(jsonString, jsonContentType).map(withCallContextHeaders) } /** @@ -265,7 +277,7 @@ object Http4sRequestAttributes { RequestScopeConnection.fromFuture(f(body, cc)).attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -307,7 +319,7 @@ object Http4sRequestAttributes { io.attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -351,7 +363,7 @@ object Http4sRequestAttributes { io.attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -389,7 +401,7 @@ object Http4sRequestAttributes { io.attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -412,7 +424,7 @@ object Http4sRequestAttributes { io.attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -477,7 +489,7 @@ object Http4sRequestAttributes { RequestScopeConnection.fromFuture(f).attempt.flatMap { case Right(result) => val jsonString = prettyRender(Extraction.decompose(result)) - Created(jsonString, jsonContentType).flatTap(recordMetric(result, _)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -498,7 +510,7 @@ object Http4sRequestAttributes { case Right((result, code)) => val jsonString = prettyRender(Extraction.decompose(result)) val status = Status.fromInt(code).getOrElse(Status.Ok) - IO.pure(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType)).flatTap(recordMetric(result, _)) + IO.pure(withCallContextHeaders(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType))).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -516,11 +528,11 @@ object Http4sRequestAttributes { result <- RequestScopeConnection.fromFuture(f(user, cc)) } yield result io.attempt.flatMap { - case Right((_, 204)) => NoContent().flatTap(recordMetric("", _)) + case Right((_, 204)) => NoContent().map(withCallContextHeaders).flatTap(recordMetric("", _)) case Right((result, code)) => val jsonString = prettyRender(Extraction.decompose(result)) val status = Status.fromInt(code).getOrElse(Status.Ok) - IO.pure(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType)).flatTap(recordMetric(result, _)) + IO.pure(withCallContextHeaders(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType))).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -532,7 +544,7 @@ object Http4sRequestAttributes { def executeDelete(req: Request[IO])(f: CallContext => Future[_]): IO[Response[IO]] = { implicit val cc: CallContext = req.callContext RequestScopeConnection.fromFuture(f(cc)).attempt.flatMap { - case Right(_) => NoContent().flatTap(recordMetric("", _)) + case Right(_) => NoContent().map(withCallContextHeaders).flatTap(recordMetric("", _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -548,7 +560,7 @@ object Http4sRequestAttributes { result <- RequestScopeConnection.fromFuture(f(user, cc)) } yield result io.attempt.flatMap { - case Right(_) => NoContent().flatTap(recordMetric("", _)) + case Right(_) => NoContent().map(withCallContextHeaders).flatTap(recordMetric("", _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } @@ -565,7 +577,7 @@ object Http4sRequestAttributes { result <- RequestScopeConnection.fromFuture(f(user, bank, cc)) } yield result io.attempt.flatMap { - case Right(_) => NoContent().flatTap(recordMetric("", _)) + case Right(_) => NoContent().map(withCallContextHeaders).flatTap(recordMetric("", _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } } diff --git a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala index 888f87a239..eec69fbecc 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala @@ -8,7 +8,7 @@ import code.api.APIFailureNewStyle import code.api.util.APIUtil.ResourceDoc import code.api.util.ErrorMessages._ import code.api.util.newstyle.ViewNewStyle -import code.api.util.{APIUtil, ApiRole, CallContext, NewStyle} +import code.api.util.{APIUtil, ApiRole, CallContext, CallContextLight, NewStyle} import code.util.Helper.MdcLoggable import com.openbankproject.commons.model._ import com.openbankproject.commons.util.ApiShortVersions @@ -361,23 +361,23 @@ object ResourceDocMiddleware extends MdcLoggable { case Right((boxUser, None)) => IO.pure(Right(ctx.copy(user = boxUser))) case Left(e: APIFailureNewStyle) => - ErrorResponseConverter.createErrorResponse(e.failCode, e.failMsg, ctx.callContext).map(Left(_)) + ErrorResponseConverter.createErrorResponse(e.failCode, e.failMsg, ctx.callContext, e.ccl).map(Left(_)) case Left(e) => // anonymousAccess threw a plain Exception(json_of_APIFailureNewStyle). // Parse the JSON to recover the original message and failCode (typically 401). // Old Style endpoints (v1.x, v2.0.0) keep 400 to match Lift Old Style behavior. // New Style endpoints (v2.1.0+) use the original failCode from the exception. - val (failMsg, parsedCode) = scala.util.Try { + val (failMsg, parsedCode, failureCallContext) = scala.util.Try { implicit val formats = org.json4s.DefaultFormats val parsed = com.openbankproject.commons.util.JsonAliases.parse(e.getMessage).extract[APIFailureNewStyle] - (parsed.failMsg, parsed.failCode) - }.getOrElse(($AuthenticatedUserIsRequired, 401)) + (parsed.failMsg, parsed.failCode, parsed.ccl) + }.getOrElse(($AuthenticatedUserIsRequired, 401, None: Option[CallContextLight])) val oldStyleShortVersions = Set("v1.2.1", "v1.3.0", "v1.4.0", "v2.0.0") val versionStr = resourceDoc.implementedInApiVersion.apiShortVersion val isOldStyle = oldStyleShortVersions.contains(versionStr) val effectiveCode = if (isOldStyle) 400 else parsedCode logger.debug(s"[ResourceDocMiddleware.authenticate] version=$versionStr isOldStyle=$isOldStyle parsedCode=$parsedCode effectiveCode=$effectiveCode") - ErrorResponseConverter.createErrorResponse(effectiveCode, failMsg, ctx.callContext).map(Left(_)) + ErrorResponseConverter.createErrorResponse(effectiveCode, failMsg, ctx.callContext, failureCallContext).map(Left(_)) } ) } diff --git a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala index ab735b1c0e..f087a47b73 100644 --- a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala +++ b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala @@ -205,7 +205,7 @@ object Http4s210 { createTransactionRequestImpl(jsonBody, user, account, ViewId(viewIdStr), transactionRequestTypeStr, cc)) } yield result).attempt.flatMap { case Right(result) => - Created(prettyRender(Extraction.decompose(result))) + Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } diff --git a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala index 1b2d0d0d68..0c5a2653fd 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala @@ -149,7 +149,7 @@ object Http4s220 { } yield result io.attempt.flatMap { case Right(result) => - Created(prettyRender(Extraction.decompose(result))) + Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } @@ -221,7 +221,7 @@ object Http4s220 { } yield result io.attempt.flatMap { case Right(result) => - Ok(prettyRender(Extraction.decompose(result))) + Ok(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } @@ -884,7 +884,7 @@ object Http4s220 { } yield result io.attempt.flatMap { case Right(result) => - Created(prettyRender(Extraction.decompose(result))) + Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } diff --git a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala index d4b483495e..4bf7f5c2dd 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala @@ -165,7 +165,7 @@ object Http4s300 { } yield result io.attempt.flatMap { case Right(result) => - Created(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))) + Created(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } @@ -235,7 +235,7 @@ object Http4s300 { } yield result io.attempt.flatMap { case Right(result) => - Ok(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))) + Ok(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) case Left(err) => code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) } diff --git a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala index 918cfc30c2..3f2e6d606f 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/RateLimitsTest.scala @@ -555,6 +555,9 @@ class RateLimitsTest extends V600ServerSetup { message should startWith(TooManyRequests) message should include("per minute") message should include(consumerId3) + And("the X-Rate-Limit headers describe the exhausted per-minute limit") + refused.headers.flatMap(h => Option(h.get("X-Rate-Limit-Limit"))) should equal(Some("2")) + refused.headers.flatMap(h => Option(h.get("X-Rate-Limit-Remaining"))) should equal(Some("0")) } finally { deleteLimit(consumerId3, id) resetCallCounters(consumerId3) From 0b47d29c723a46c5b2a769c9eae2d9e6e87e97d9 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 8 Sep 2026 14:11:50 +0200 Subject: [PATCH 13/13] Refactor to use EndpointHelpers.withUserAndBankCreated --- .../main/scala/code/api/OBPRestHelper.scala | 4 +- .../main/scala/code/api/util/APIUtil.scala | 6 +-- .../main/scala/code/api/util/ErrorUtil.scala | 4 +- .../util/http4s/ErrorResponseConverter.scala | 18 +++---- .../code/api/util/http4s/Http4sSupport.scala | 19 +++++++ .../util/http4s/ResourceDocMiddleware.scala | 4 +- .../scala/code/api/v2_1_0/Http4s210.scala | 36 ++++++------- .../scala/code/api/v2_2_0/Http4s220.scala | 52 ++++--------------- .../scala/code/api/v3_0_0/Http4s300.scala | 36 +++---------- 9 files changed, 70 insertions(+), 109 deletions(-) diff --git a/obp-api/src/main/scala/code/api/OBPRestHelper.scala b/obp-api/src/main/scala/code/api/OBPRestHelper.scala index ccad0101ae..f2cbfd0228 100644 --- a/obp-api/src/main/scala/code/api/OBPRestHelper.scala +++ b/obp-api/src/main/scala/code/api/OBPRestHelper.scala @@ -76,13 +76,13 @@ object APIFailure { case class APIFailureNewStyle(failMsg: String, failCode: Int = 400, - ccl: Option[CallContextLight] = None + callContextLight: Option[CallContextLight] = None ){ def translatedErrorMessage = { val errorCode = extractErrorMessageCode(failMsg) val errorBody = extractErrorMessageBody(failMsg) - val localeUrlParameter = getHttpRequestUrlParam(ccl.map(_.url).getOrElse(""), PARAM_LOCALE) + val localeUrlParameter = getHttpRequestUrlParam(callContextLight.map(_.url).getOrElse(""), PARAM_LOCALE) val localeFromUrl = I18NUtil.computeLocale(localeUrlParameter) val locale: Locale = if (localeFromUrl.toString.equals("")) I18NUtil.getDefaultLocale() diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 8ffde2526c..484207481b 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -3243,12 +3243,12 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ Failure (m, e, c) ?~! af.translatedErrorMessage } val failuresMsg = filterMessage(obj) - val callContext = af.ccl.map(_.copy(httpCode = Some(af.failCode))) - val apiFailure = af.copy(failMsg = failuresMsg).copy(ccl = callContext) + val callContext = af.callContextLight.map(_.copy(httpCode = Some(af.failCode))) + val apiFailure = af.copy(failMsg = failuresMsg).copy(callContextLight = callContext) throw new Exception(com.openbankproject.commons.util.JsonAliases.compactRender(Extraction.decompose(apiFailure))) case ParamFailure(_, _, _, failure : APIFailure) => val callContext = CallContextLight() - val apiFailure = APIFailureNewStyle(failMsg = failure.msg, failCode = failure.responseCode, ccl = Some(callContext)) + val apiFailure = APIFailureNewStyle(failMsg = failure.msg, failCode = failure.responseCode, callContextLight = Some(callContext)) throw new Exception(com.openbankproject.commons.util.JsonAliases.compactRender(Extraction.decompose(apiFailure))) case ParamFailure(msg,_,_,_) => throw new Exception(msg) diff --git a/obp-api/src/main/scala/code/api/util/ErrorUtil.scala b/obp-api/src/main/scala/code/api/util/ErrorUtil.scala index a1735f7821..691fdb852b 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorUtil.scala @@ -14,7 +14,7 @@ object ErrorUtil { val apiFailure = APIFailureNewStyle( failMsg = errorMessage, failCode = httpCode, - ccl = second.map(_.toLight) + callContextLight = second.map(_.toLight) ) val failureBox = Empty ~> apiFailure ( @@ -27,7 +27,7 @@ object ErrorUtil { val apiFailure = APIFailureNewStyle( failMsg = errorMessage, failCode = httpCode, - ccl = cc.map(_.toLight) + callContextLight = cc.map(_.toLight) ) val failureBox: Box[T] = Empty ~> apiFailure fullBoxOrException(failureBox) diff --git a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala index 1ebbf7b797..c80661d382 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala @@ -48,9 +48,9 @@ object ErrorResponseConverter { * stamps the exhausted limit and its reset time there, while the CallContext the middleware still * holds predates the rate-limit check. */ - private def withResponseHeaders(response: Response[IO], callContext: CallContext, ccl: Option[CallContextLight]): Response[IO] = { + private def withResponseHeaders(response: Response[IO], callContext: CallContext, callContextLight: Option[CallContextLight]): Response[IO] = { val withCorrelationId = response.putHeaders(Header.Raw(CIString("Correlation-Id"), callContext.correlationId)) - code.api.util.APIUtil.getHeadersNewStyle(Some(ccl.getOrElse(callContext.toLight))).list.foldLeft(withCorrelationId) { + code.api.util.APIUtil.getHeadersNewStyle(Some(callContextLight.getOrElse(callContext.toLight))).list.foldLeft(withCorrelationId) { case (r, (name, value)) => r.putHeaders(Header.Raw(CIString(name), value)) } } @@ -61,10 +61,10 @@ object ErrorResponseConverter { val msg = Option(error.getMessage).getOrElse("").trim if (msg.startsWith("{") && msg.contains("\"failCode\"") && msg.contains("\"failMsg\"")) { try { - val jv = parse(msg) - val failCode = (jv \ "failCode").extract[Int] - val failMsg = (jv \ "failMsg").extract[String] - Some(APIFailureNewStyle(failMsg, failCode, (jv \ "ccl").extractOpt[CallContextLight])) + val jsonValue = parse(msg) + val failCode = (jsonValue \ "failCode").extract[Int] + val failMsg = (jsonValue \ "failMsg").extract[String] + Some(APIFailureNewStyle(failMsg, failCode, (jsonValue \ "callContextLight").extractOpt[CallContextLight])) } catch { case _: Throwable => None } @@ -162,7 +162,7 @@ object ErrorResponseConverter { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(resolvedCode, failure.failMsg, callContext) else toJsonString(OBPErrorResponse(resolvedCode, failure.failMsg)) val status = org.http4s.Status.fromInt(resolvedCode).getOrElse(org.http4s.Status.BadRequest) - IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, failure.ccl)) + IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, failure.callContextLight)) } /** @@ -190,10 +190,10 @@ object ErrorResponseConverter { /** * Create error response with specific status code and message. */ - def createErrorResponse(statusCode: Int, message: String, callContext: CallContext, ccl: Option[CallContextLight] = None): IO[Response[IO]] = { + def createErrorResponse(statusCode: Int, message: String, callContext: CallContext, callContextLight: Option[CallContextLight] = None): IO[Response[IO]] = { val body = if (isBerlinGroupRequest(callContext)) toBgErrorBody(statusCode, message, callContext) else toJsonString(OBPErrorResponse(statusCode, message)) val status = org.http4s.Status.fromInt(statusCode).getOrElse(org.http4s.Status.BadRequest) - IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, ccl)) + IO.pure(withResponseHeaders(Response[IO](status).withEntity(body).withContentType(jsonContentType), callContext, callContextLight)) } } diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index f3483fd307..b2dde552a4 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -238,6 +238,25 @@ object Http4sRequestAttributes { } } + /** + * Execute business logic requiring both User and Bank, without a typed body. + * Returns 201 Created on success, converts errors via ErrorResponseConverter. + */ + def withUserAndBankCreated[A](req: Request[IO])(f: (User, Bank, CallContext) => Future[A])(implicit formats: Formats): IO[Response[IO]] = { + implicit val cc: CallContext = req.callContext + val io = for { + user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) + bank <- IO.fromOption(cc.bank)(new RuntimeException("Bank not found in CallContext")) + result <- RequestScopeConnection.fromFuture(f(user, bank, cc)) + } yield result + io.attempt.flatMap { + case Right(result) => + val jsonString = prettyRender(Extraction.decompose(result)) + Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) + case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) + } + } + /** * Parse the request body from CallContext into type B. * Returns Left(error message) if body is absent or not valid JSON for B. diff --git a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala index eec69fbecc..8fccebe8bb 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala @@ -361,7 +361,7 @@ object ResourceDocMiddleware extends MdcLoggable { case Right((boxUser, None)) => IO.pure(Right(ctx.copy(user = boxUser))) case Left(e: APIFailureNewStyle) => - ErrorResponseConverter.createErrorResponse(e.failCode, e.failMsg, ctx.callContext, e.ccl).map(Left(_)) + ErrorResponseConverter.createErrorResponse(e.failCode, e.failMsg, ctx.callContext, e.callContextLight).map(Left(_)) case Left(e) => // anonymousAccess threw a plain Exception(json_of_APIFailureNewStyle). // Parse the JSON to recover the original message and failCode (typically 401). @@ -370,7 +370,7 @@ object ResourceDocMiddleware extends MdcLoggable { val (failMsg, parsedCode, failureCallContext) = scala.util.Try { implicit val formats = org.json4s.DefaultFormats val parsed = com.openbankproject.commons.util.JsonAliases.parse(e.getMessage).extract[APIFailureNewStyle] - (parsed.failMsg, parsed.failCode, parsed.ccl) + (parsed.failMsg, parsed.failCode, parsed.callContextLight) }.getOrElse(($AuthenticatedUserIsRequired, 401, None: Option[CallContextLight])) val oldStyleShortVersions = Set("v1.2.1", "v1.3.0", "v1.4.0", "v2.0.0") val versionStr = resourceDoc.implementedInApiVersion.apiShortVersion diff --git a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala index f087a47b73..b075640960 100644 --- a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala +++ b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala @@ -39,7 +39,7 @@ import com.openbankproject.commons.model._ import com.openbankproject.commons.model.enums.TransactionRequestTypes._ import com.openbankproject.commons.model.enums.{ChallengeType, SuppliedAnswerType, TransactionRequestTypes} import com.openbankproject.commons.util.{ApiVersion, ApiVersionStatus, ScannedApiVersion} -import net.liftweb.common.{Failure, Full} +import net.liftweb.common.{Box, Failure, Full} import com.openbankproject.commons.util.JsonAliases.{compactRender, prettyRender} import org.json4s.JsonDSL._ import org.json4s.{Extraction, Formats} @@ -188,26 +188,20 @@ object Http4s210 { val createTransactionRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "banks" / _ / "accounts" / _ / viewIdStr / "transaction-request-types" / transactionRequestTypeStr / "transaction-requests" => - implicit val cc: CallContext = req.callContext - // Use cc.httpBody (cached by ResourceDocMiddleware via cachedBodyKey) instead of re-reading - // req.bodyText, which is empty after the bridge cascade has already consumed the stream. - (for { - // Check type validity before requiring middleware-resolved entities: for an invalid - // type the middleware finds no matching ResourceDoc and skips bankAccount resolution, - // so cc.bankAccount is None — checking the type first avoids a misleading AccountNotFound. - _ <- if (v210SupportedTransactionRequestTypes.contains(transactionRequestTypeStr)) IO.unit - else IO.raiseError(new RuntimeException(liftWrite(code.api.APIFailureNewStyle( - s"$InvalidTransactionRequestType: '$transactionRequestTypeStr'", 400, Some(cc.toLight))))) - jsonBody <- IO.pure(cc.httpBody.getOrElse("")) - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - account <- IO.fromOption(cc.bankAccount)(new RuntimeException(AccountNotFound)) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - createTransactionRequestImpl(jsonBody, user, account, ViewId(viewIdStr), transactionRequestTypeStr, cc)) - } yield result).attempt.flatMap { - case Right(result) => - Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + // Check the type before requiring middleware-resolved entities: for an invalid type the + // middleware finds no matching ResourceDoc and skips bankAccount resolution, so cc.bankAccount + // is None; checking the type first avoids a misleading AccountNotFound. cc.httpBody is the body + // cached by ResourceDocMiddleware; req.bodyText is empty once the bridge cascade consumed it. + EndpointHelpers.executeFutureCreated(req) { + val cc: CallContext = req.callContext + for { + _ <- code.util.Helper.booleanToFuture(s"$InvalidTransactionRequestType: '$transactionRequestTypeStr'", cc = Some(cc)) { + v210SupportedTransactionRequestTypes.contains(transactionRequestTypeStr) + } + user <- Future(unboxFullOrFail(cc.user, Some(cc), AuthenticatedUserIsRequired, 401)) + account <- Future(unboxFullOrFail(Box(cc.bankAccount), Some(cc), AccountNotFound)) + result <- createTransactionRequestImpl(cc.httpBody.getOrElse(""), user, account, ViewId(viewIdStr), transactionRequestTypeStr, cc) + } yield result } } diff --git a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala index 0c5a2653fd..97a424c6b7 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala @@ -137,21 +137,12 @@ object Http4s220 { // VIEW_ACCOUNT_ID (non-standard name) bypasses middleware account-existence check so the // handler can return 400 (not 404) for a missing account, matching Lift behaviour. case req @ POST -> `prefixPath` / "banks" / _ / "accounts" / accountIdStr / "views" => - implicit val cc: CallContext = req.callContext - val io = for { - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - bank <- IO.fromOption(cc.bank)(new RuntimeException(BankNotFound)) - rawBox <- IO.fromFuture(IO(Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)).map(_._1))) - account <- IO(unboxFullOrFail(rawBox, Some(cc), BankAccountNotFound)) - body <- IO.pure(cc.httpBody.getOrElse("")) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - createViewImpl(user, account, body, cc)) - } yield result - io.attempt.flatMap { - case Right(result) => - Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + EndpointHelpers.withUserAndBankCreated(req) { (user, bank, cc) => + for { + (rawBox, _) <- Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)) + account <- Future(unboxFullOrFail(rawBox, Some(cc), BankAccountNotFound)) + result <- createViewImpl(user, account, cc.httpBody.getOrElse(""), cc) + } yield result } } @@ -211,19 +202,8 @@ object Http4s220 { val updateViewForBankAccount: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "banks" / _ / "accounts" / _ / "views" / viewIdStr => - implicit val cc: CallContext = req.callContext - val io = for { - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - account <- IO.fromOption(cc.bankAccount)(new RuntimeException(AccountNotFound)) - body <- IO.pure(cc.httpBody.getOrElse("")) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - updateViewImpl(user, account, ViewId(viewIdStr), body, cc)) - } yield result - io.attempt.flatMap { - case Right(result) => - Ok(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + EndpointHelpers.withBankAccount(req) { (user, account, cc) => + updateViewImpl(user, account, ViewId(viewIdStr), cc.httpBody.getOrElse(""), cc) } } @@ -873,20 +853,8 @@ object Http4s220 { val createCounterparty: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "banks" / _ / "accounts" / _ / _ / "counterparties" => - implicit val cc: CallContext = req.callContext - val io = for { - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - account <- IO.fromOption(cc.bankAccount)(new RuntimeException(AccountNotFound)) - view <- IO.fromOption(cc.view)(new RuntimeException(ViewNotFound)) - body <- IO.pure(cc.httpBody.getOrElse("")) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - createCounterpartyImpl(user, account, view, body, cc)) - } yield result - io.attempt.flatMap { - case Right(result) => - Created(prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + EndpointHelpers.withViewCreated(req) { (user, account, view, cc) => + createCounterpartyImpl(user, account, view, cc.httpBody.getOrElse(""), cc) } } diff --git a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala index 4bf7f5c2dd..71e1d7d3c0 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala @@ -153,21 +153,12 @@ object Http4s300 { val createViewForBankAccount: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "banks" / _ / "accounts" / accountIdStr / "views" => - implicit val cc: CallContext = req.callContext - val io = for { - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - bank <- IO.fromOption(cc.bank)(new RuntimeException(BankNotFound)) - rawBox <- IO.fromFuture(IO(Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)).map(_._1))) - account <- IO(unboxFullOrFail(rawBox, Some(cc), BankAccountNotFound, 404)) - body <- IO.pure(cc.httpBody.getOrElse("")) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - createViewImpl300(user, account, body, cc)) - } yield result - io.attempt.flatMap { - case Right(result) => - Created(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + EndpointHelpers.withUserAndBankCreated(req) { (user, bank, cc) => + for { + (rawBox, _) <- Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)) + account <- Future(unboxFullOrFail(rawBox, Some(cc), BankAccountNotFound, 404)) + result <- createViewImpl300(user, account, cc.httpBody.getOrElse(""), cc) + } yield result } } @@ -225,19 +216,8 @@ object Http4s300 { val updateViewForBankAccount: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "banks" / _ / "accounts" / _ / "views" / viewIdStr => - implicit val cc: CallContext = req.callContext - val io = for { - user <- IO.fromOption(cc.user.toOption)(new RuntimeException(AuthenticatedUserIsRequired)) - account <- IO.fromOption(cc.bankAccount)(new RuntimeException(AccountNotFound)) - body <- IO.pure(cc.httpBody.getOrElse("")) - result <- code.api.util.http4s.RequestScopeConnection.fromFuture( - updateViewImpl300(user, account, ViewId(viewIdStr), body, cc)) - } yield result - io.attempt.flatMap { - case Right(result) => - Ok(com.openbankproject.commons.util.JsonAliases.prettyRender(Extraction.decompose(result))).map(EndpointHelpers.withCallContextHeaders) - case Left(err) => - code.api.util.http4s.ErrorResponseConverter.toHttp4sResponse(err, cc) + EndpointHelpers.withBankAccount(req) { (user, account, cc) => + updateViewImpl300(user, account, ViewId(viewIdStr), cc.httpBody.getOrElse(""), cc) } }