[OPIK-8040] [BE] fix: apply online evaluation sampling to production traces only

apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluator.java

@@ -12,6 +12,8 @@
 import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.DecimalMax;
+import jakarta.validation.constraints.DecimalMin;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
 import jakarta.validation.constraints.Size;
@@ -80,7 +82,8 @@ public abstract sealed class AutomationRuleEvaluator<T, E extends Filter> implem
     @NotBlank @Size(max = 150, message = "cannot exceed 150 characters") private final String name;
 
     @JsonView({View.Public.class, View.Write.class})
-    private final float samplingRate;
+    @Schema(description = "Fraction of production (SDK-logged) items this rule scores, from 0 to 1. Trace rules ignore this value for experiment, playground and optimization traces and score them in full; span and thread rules only ever evaluate SDK-logged data.")
+    @DecimalMin("0") @DecimalMax("1") private final float samplingRate;
 
     @JsonView({View.Public.class, View.Write.class})
     @Builder.Default

apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorUpdate.java

@@ -10,6 +10,8 @@
 import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.DecimalMax;
+import jakarta.validation.constraints.DecimalMin;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
 import jakarta.validation.constraints.Size;
@@ -54,7 +56,8 @@ public abstract sealed class AutomationRuleEvaluatorUpdate<T, E extends Filter>
     // the API boundary instead of failing the update (OPIK-7371).
     @NotBlank @Size(max = 150, message = "cannot exceed 150 characters") private final String name;
 
-    private final float samplingRate;
+    @Schema(description = "Fraction of production (SDK-logged) items this rule scores, from 0 to 1. Trace rules ignore this value for experiment, playground and optimization traces and score them in full; span and thread rules only ever evaluate SDK-logged data.")
+    @DecimalMin("0") @DecimalMax("1") private final float samplingRate;
 
     @Builder.Default
     private final boolean enabled = true;

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/events/OnlineScoringSampler.java

@@ -127,11 +127,19 @@ private void recordDecision(String workspaceId, String workspaceName, Automation
     private boolean skip(String workspaceId, String workspaceName, AutomationRuleEvaluator<?, ?> evaluator, Trace trace,
             String decision, String message, Object... args) {
         recordDecision(workspaceId, workspaceName, evaluator, decision, 1);
+        logForUser(workspaceId, evaluator, trace, message, args);
+        return false;
+    }
+
+    /**
+     * Emits one line on the rule's user-facing log stream for the given trace.
+     */
+    private void logForUser(String workspaceId, AutomationRuleEvaluator<?, ?> evaluator, Trace trace,
+            String message, Object... args) {
         // Important to set the workspaceId for logging purposes
         try (var logContext = createTraceLoggingContext(workspaceId, evaluator, trace)) {
             userFacingLogger.info(message, args);
         }
-        return false;
     }
 
     /**
@@ -354,6 +362,15 @@ private boolean shouldSampleTrace(AutomationRuleEvaluator<?, ?> evaluator, Strin
                     trace.id(), evaluator.getName());
         }
 
+        // The sampling rate thins a production stream, so it applies to SDK traces only.
+        if (!Source.isLoggingSource(trace.source())) {
+            logForUser(workspaceId, evaluator, trace,
+                    "The traceId '{}' is not subject to the sampling rate '{}' for rule: '{}',"
+                            + " as the rate applies to production traces only",
+                    trace.id(), evaluator.getSamplingRate(), evaluator.getName());
+            return true;
+        }
+
         if (secureRandom.nextFloat() >= evaluator.getSamplingRate()) {
             return skip(workspaceId, workspaceName, evaluator, trace, DECISION_SKIPPED_SAMPLING,
                     "The traceId '{}' was skipped for rule: '{}' and per the sampling rate '{}'",

apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/events/OnlineScoringSamplerTest.java

@@ -132,15 +132,7 @@ void processesPythonEvaluatorWhenToggleEnabled() {
 
             onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
 
-            var expectedMessage = TraceToScoreUserDefinedMetricPython.builder()
-                    .trace(trace)
-                    .ruleId(evaluator.getId())
-                    .ruleName(evaluator.getName())
-                    .code(evaluator.getCode())
-                    .workspaceId(workspaceId)
-                    .userName(userName)
-                    .build();
-            verify(onlineScorePublisher).enqueueMessage(List.of(expectedMessage),
+            verify(onlineScorePublisher).enqueueMessage(List.of(toPythonMessage(evaluator, trace)),
                     AutomationRuleEvaluatorType.USER_DEFINED_METRIC_PYTHON);
         }
 
@@ -475,6 +467,87 @@ void samplesAllTracesWhenSamplingRateIsOne() {
                     List.of(toLlmMessage(evaluator, trace1), toLlmMessage(evaluator, trace2)),
                     AutomationRuleEvaluatorType.LLM_AS_JUDGE);
         }
+
+        @Test
+        void scoresExperimentTracesWhenSamplingRateIsZero() {
+            var trace = createTrace(Source.EXPERIMENT);
+            var evaluator = createLlmEvaluator(true, 0.0f, List.of());
+            whenFindAllLlmEvaluators(evaluator);
+
+            onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
+
+            verify(onlineScorePublisher).enqueueMessage(List.of(toLlmMessage(evaluator, trace)),
+                    AutomationRuleEvaluatorType.LLM_AS_JUDGE);
+        }
+
+        @Test
+        void scoresExperimentTracesButSkipsProductionTracesWhenSamplingRateIsZero() {
+            var sdkTrace = createTrace(Source.SDK);
+            var experimentTrace = createTrace(Source.EXPERIMENT);
+            var evaluator = createLlmEvaluator(true, 0.0f, List.of());
+            whenFindAllLlmEvaluators(evaluator);
+
+            onlineScoringSampler
+                    .onTracesCreated(new TracesCreated(List.of(sdkTrace, experimentTrace), workspaceId, userName));
+
+            verify(onlineScorePublisher).enqueueMessage(List.of(toLlmMessage(evaluator, experimentTrace)),
+                    AutomationRuleEvaluatorType.LLM_AS_JUDGE);
+        }
+
+        @ParameterizedTest
+        @EnumSource(value = Source.class, mode = EnumSource.Mode.EXCLUDE, names = {"SDK", "EXPERIMENT"})
+        void scoresSelectedRuleTracesWhenSamplingRateIsZero(Source source) {
+            var evaluator = createLlmEvaluator(true, 0.0f, List.of());
+            var trace = createTrace(source).toBuilder()
+                    .metadata(metadataWithRuleIds(evaluator.getId()))
+                    .build();
+            whenFindAllLlmEvaluators(evaluator);
+
+            onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
+
+            verify(onlineScorePublisher).enqueueMessage(List.of(toLlmMessage(evaluator, trace)),
+                    AutomationRuleEvaluatorType.LLM_AS_JUDGE);
+        }
+
+        @Test
+        void scoresExperimentTracesWhenSamplingRateIsZeroForPythonEvaluator() {
+            when(serviceTogglesConfig.isPythonEvaluatorEnabled()).thenReturn(true);
+            var trace = createTrace(Source.EXPERIMENT);
+            var evaluator = createPythonEvaluator(0.0f);
+            whenFindAllPythonEvaluators(evaluator);
+
+            onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
+
+            verify(onlineScorePublisher).enqueueMessage(List.of(toPythonMessage(evaluator, trace)),
+                    AutomationRuleEvaluatorType.USER_DEFINED_METRIC_PYTHON);
+        }
+
+        @Test
+        void stillHonoursFiltersOnExperimentTracesWhenSamplingIsBypassed() {
+            var trace = createTrace(Source.EXPERIMENT).toBuilder().name("no-match").build();
+            var filter = TraceFilter.builder()
+                    .field(TraceField.NAME)
+                    .operator(Operator.EQUAL)
+                    .value("expected-name")
+                    .build();
+            var evaluator = createLlmEvaluator(true, 0.0f, List.of(filter));
+            whenFindAllLlmEvaluators(evaluator);
+
+            onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
+
+            verify(onlineScorePublisher, never()).enqueueMessage(any(), any());
+        }
+
+        @Test
+        void stillHonoursDisabledRuleOnExperimentTracesWhenSamplingIsBypassed() {
+            var trace = createTrace(Source.EXPERIMENT);
+            var evaluator = createLlmEvaluator(false, 0.0f, List.of());
+            whenFindAllLlmEvaluators(evaluator);
+
+            onlineScoringSampler.onTracesCreated(new TracesCreated(List.of(trace), workspaceId, userName));
+
+            verify(onlineScorePublisher, never()).enqueueMessage(any(), any());
+        }
     }
 
     @Nested
@@ -732,6 +805,18 @@ private TraceToScoreLlmAsJudge toLlmMessage(AutomationRuleEvaluatorLlmAsJudge ev
                 .build();
     }
 
+    private TraceToScoreUserDefinedMetricPython toPythonMessage(
+            AutomationRuleEvaluatorUserDefinedMetricPython evaluator, Trace trace) {
+        return TraceToScoreUserDefinedMetricPython.builder()
+                .trace(trace)
+                .ruleId(evaluator.getId())
+                .ruleName(evaluator.getName())
+                .code(evaluator.getCode())
+                .workspaceId(workspaceId)
+                .userName(userName)
+                .build();
+    }
+
     private ObjectNode metadataWithRuleIds(UUID... ruleIds) {
         var metadata = JsonUtils.createObjectNode();
         var array = metadata.putArray("selected_rule_ids");

apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/AutomationRuleEvaluatorsResourceTest.java

@@ -982,6 +982,73 @@ void updateEvaluator__whenNameExceedsColumnLength__thenReturnUnprocessableEntity
             }
         }
 
+        @ParameterizedTest
+        @ValueSource(floats = {-0.1f, 1.1f})
+        @DisplayName("create evaluator: when the sampling rate is outside [0,1], then reject at the API boundary")
+        void createEvaluator__whenSamplingRateOutOfRange__thenReturnUnprocessableEntity(float samplingRate) {
+            var projectId = createProject();
+            var evaluator = factory.manufacturePojo(AutomationRuleEvaluatorLlmAsJudge.class).toBuilder()
+                    .samplingRate(samplingRate)
+                    .projectIds(Set.of(projectId))
+                    .build();
+
+            try (var response = evaluatorsResourceClient.createEvaluator(
+                    evaluator, WORKSPACE_NAME, API_KEY, HttpStatus.SC_UNPROCESSABLE_ENTITY)) {
+                assertThat(response.readEntity(com.comet.opik.api.error.ErrorMessage.class).errors())
+                        .anyMatch(error -> error.contains("samplingRate"));
+            }
+        }
+
+        @ParameterizedTest
+        @ValueSource(floats = {0f, 1f})
+        @DisplayName("create evaluator: the inclusive sampling rate bounds are accepted")
+        void createEvaluator__whenSamplingRateOnBounds__thenSucceed(float samplingRate) {
+            var projectId = createProject();
+            var evaluator = factory.manufacturePojo(AutomationRuleEvaluatorLlmAsJudge.class).toBuilder()
+                    .samplingRate(samplingRate)
+                    .projectIds(Set.of(projectId))
+                    .build();
+
+            assertThat(evaluatorsResourceClient.createEvaluator(evaluator, WORKSPACE_NAME, API_KEY)).isNotNull();
+        }
+
+        @ParameterizedTest
+        @ValueSource(floats = {-0.1f, 1.1f})
+        @DisplayName("update evaluator: when the sampling rate is outside [0,1], then reject at the API boundary")
+        void updateEvaluator__whenSamplingRateOutOfRange__thenReturnUnprocessableEntity(float samplingRate) {
+            var projectId = createProject();
+            var id = createLlmRule("Resample me " + UUID.randomUUID(), projectId);
+
+            var update = factory.manufacturePojo(AutomationRuleEvaluatorUpdateLlmAsJudge.class).toBuilder()
+                    .samplingRate(samplingRate)
+                    .projectIds(Set.of(projectId))
+                    .build();
+
+            try (var response = evaluatorsResourceClient.callUpdateEvaluator(id, WORKSPACE_NAME, update, API_KEY)) {
+                assertThat(response.getStatusInfo().getStatusCode())
+                        .isEqualTo(HttpStatus.SC_UNPROCESSABLE_ENTITY);
+                assertThat(response.readEntity(com.comet.opik.api.error.ErrorMessage.class).errors())
+                        .anyMatch(error -> error.contains("samplingRate"));
+            }
+        }
+
+        @ParameterizedTest
+        @ValueSource(floats = {0f, 1f})
+        @DisplayName("update evaluator: the inclusive sampling rate bounds are accepted")
+        void updateEvaluator__whenSamplingRateOnBounds__thenSucceed(float samplingRate) {
+            var projectId = createProject();
+            var id = createLlmRule("Resample me " + UUID.randomUUID(), projectId);
+
+            var update = factory.manufacturePojo(AutomationRuleEvaluatorUpdateLlmAsJudge.class).toBuilder()
+                    .samplingRate(samplingRate)
+                    .projectIds(Set.of(projectId))
+                    .build();
+
+            try (var response = evaluatorsResourceClient.callUpdateEvaluator(id, WORKSPACE_NAME, update, API_KEY)) {
+                assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_NO_CONTENT);
+            }
+        }
+
         @Test
         @DisplayName("a rule literally named '%' does not swallow unrelated names in the same project")
         void whenAWildcardNamedRuleExists__thenUnrelatedNamesAreNotSuffixed() {

apps/opik-documentation/documentation/fern/docs-v2/production/online-evaluation/rules.mdx

@@ -34,7 +34,7 @@ To create a new scoring metric in the UI, first navigate to the project you woul
 When creating a new rule, you will be presented with the following options:
 
 1. **Name:** The name of the rule
-2. **Sampling rate:** The percentage of traces to score. When set to `100%`, all traces will be scored.
+2. **Sampling rate:** The percentage of production traces to score. When set to `100%`, all production traces will be scored. For trace rules the rate applies to production (SDK-logged) traces only: traces from experiments ignore it and are always scored when the rule matches, and traces from the playground or an optimization run ignore it too but are only evaluated when you explicitly select the rule for that run. Thread and span rules only ever run on production (SDK-logged) data — threads and spans from experiments, the playground and optimization runs are excluded from online evaluation entirely.
 3. **Model:** The model to use to run the LLM as a Judge metric. For evaluating traces with images, make sure to select a model that supports vision capabilities.
 4. **Prompt:** The LLM as a Judge prompt to use. Opik provides a set of base prompts (Hallucination, Moderation, Answer Relevance) that you can use or you can define your own. Variables in the prompt should be in `{{variable_name}}` format.
 5. **Variable mapping:** This is the mapping of the variables in the prompt to the values from the trace.

apps/opik-documentation/documentation/fern/docs/production/rules.mdx

@@ -35,7 +35,7 @@ To create a new scoring metric in the UI, first navigate to the project you woul
 When creating a new rule, you will be presented with the following options:
 
 1. **Name:** The name of the rule
-2. **Sampling rate:** The percentage of traces to score. When set to `100%`, all traces will be scored.
+2. **Sampling rate:** The percentage of production traces to score. When set to `100%`, all production traces will be scored. The rate applies to production (SDK-logged) traces only. Traces from experiments ignore it and are always scored when the rule matches. Traces from the playground or an optimization run ignore it too, but are only evaluated when you explicitly select the rule for that run.
 3. **Model:** The model to use to run the LLM as a Judge metric. For evaluating traces with images, make sure to select a model that supports vision capabilities.
 4. **Prompt:** The LLM as a Judge prompt to use. Opik provides a set of base prompts (Hallucination, Moderation, Answer Relevance) that you can use or you can define your own. Variables in the prompt should be in `{{variable_name}}` format.
 5. **Variable mapping:** This is the mapping of the variables in the prompt to the values from the trace.

apps/opik-frontend/src/v2/pages-shared/automations/AddEditRuleDialog/RuleFilteringSection.tsx

@@ -553,7 +553,13 @@ const RuleFilteringSection: React.FC<RuleFilteringSectionProps> = ({
                   }
                   id="sampling_rate"
                   label="Sampling rate"
-                  tooltip="Percentage of traces to evaluate"
+                  tooltip={
+                    isTraceScope
+                      ? "Percentage of production (SDK-logged) traces to evaluate. Traces from experiments, the playground and optimization runs ignore this rate."
+                      : `Percentage of production (SDK-logged) ${
+                          isThreadScope ? "threads" : "spans"
+                        } to evaluate. Only SDK-logged data is evaluated.`
+                  }
                   suffix="%"
                 />
               )}