fix: keep LATERAL ON clauses on alias output names

requalify_filter mapped both the pulled-up correlation predicate and the
user's ON clause through the pre-alias schema. ON is already bound to the
alias output. When an inner column shares that qualifier, the map sends
ON s.id to a later id:1. Pull-up only appends columns, and those are the
only names unique_field_aliases suffixes, so the planner's ON names stay
valid. EXISTS, IN, and scalar decorrelation only positionally rewrite
correlation columns, not a user predicate that already uses output names.

Assisted by Cursor (Grok).

Author: canyang25

datafusion/optimizer/src/decorrelate_lateral_join.rs

@@ -135,25 +135,27 @@ fn rewrite_internal(join: Join) -> Result<Transformed<LogicalPlan>> {
         .cloned();
 
     // Re-wrap in SubqueryAlias if the original had one, preserving the alias name.
-    // The SubqueryAlias re-qualifies all columns with the alias, so we must also
-    // rewrite column references in both the correlation and ON-clause filters.
+    // The correlation filter still names the inner tables. `try_new` may suffix
+    // a duplicate, so those columns are mapped by position.
+    //
+    // The user's ON clause already names the alias output. Pull-up only appends
+    // columns, and `unique_field_aliases` only suffixes those new columns, so
+    // the names ON uses do not change. Mapping ON through the pre-alias schema
+    // is wrong when an inner column shares the alias qualifier (`l AS s` inside
+    // `LATERAL (...) s`): `s.id` would move from the first output `id` to a
+    // later `id:1`.
     let (right_plan, correlation_filter, original_join_filter) =
         if let Some(ref alias) = alias {
             let input_schema = Arc::clone(rewritten_subquery.schema());
             let right = LogicalPlan::SubqueryAlias(SubqueryAlias::try_new(
                 Arc::new(rewritten_subquery),
                 alias.clone(),
             )?);
-            // `try_new` may suffix a duplicate (`id`, then `id:1`). Map from
-            // the schema it consumed to the schema it produced.
             let output_schema = Arc::clone(right.schema());
             let corr = correlation_filter
                 .map(|f| requalify_filter(f, &input_schema, &output_schema))
                 .transpose()?;
-            let on = original_join_filter
-                .map(|f| requalify_filter(f, &input_schema, &output_schema))
-                .transpose()?;
-            (right, corr, on)
+            (right, corr, original_join_filter)
         } else {
             (rewritten_subquery, correlation_filter, original_join_filter)
         };

datafusion/sqllogictest/test_files/lateral_join.slt

@@ -1093,6 +1093,57 @@ DROP TABLE l;
 statement ok
 DROP TABLE m;
 
+# The user's ON clause names the alias output (`s.id` is decoy.id, the
+# first id). The inner table is also `s`, so a positional map of that
+# ON predicate follows the later field and filters `l` instead.
+statement ok
+CREATE TABLE o(k INT) AS VALUES (1), (100);
+
+statement ok
+CREATE TABLE l(id INT, v INT) AS VALUES (1, 10), (100, 99);
+
+statement ok
+CREATE TABLE decoy(id INT) AS VALUES (100);
+
+query IIII
+SELECT o.k, s.id, s."id:1", s.v
+FROM o
+JOIN LATERAL (
+  SELECT decoy.id, s.id, s.v
+  FROM l AS s, decoy
+  WHERE s.id = o.k
+) s
+ON s.id = 100
+ORDER BY o.k, s.v;
+----
+1 100 1 10
+100 100 100 99
+
+# `s.id` in ON is decoy.id, which is 100, so both correlated rows pass.
+# Remapping that ON onto l.id turns the k=1 row into NULLs.
+query IIII
+SELECT o.k, s.id, s."id:1", s.v
+FROM o
+LEFT JOIN LATERAL (
+  SELECT decoy.id, s.id, s.v
+  FROM l AS s, decoy
+  WHERE s.id = o.k
+) s
+ON s.id = 100
+ORDER BY o.k, s.v;
+----
+1 100 1 10
+100 100 100 99
+
+statement ok
+DROP TABLE o;
+
+statement ok
+DROP TABLE l;
+
+statement ok
+DROP TABLE decoy;
+
 ###########################################################
 # Cleanup
 ###########################################################

datafusion/sqllogictest/test_files/subquery.slt

@@ -3407,3 +3407,67 @@ DROP TABLE l;
 
 statement ok
 DROP TABLE decoy;
+
+# The IN value is decoy_hit.id (1). The correlation is l_miss.id (99),
+# which matches none of 1, 2, 3, so IN is empty and NOT IN / NOT EXISTS
+# keep every outer row.
+statement ok
+CREATE TABLE o(k INT) AS VALUES (1), (2), (3);
+
+statement ok
+CREATE TABLE l_miss(id INT) AS VALUES (99);
+
+statement ok
+CREATE TABLE decoy_hit(id INT) AS VALUES (1);
+
+query I
+SELECT o.k FROM o WHERE o.k IN (
+  SELECT decoy_hit.id FROM l_miss, decoy_hit WHERE l_miss.id = o.k
+) ORDER BY o.k;
+----
+
+query I
+SELECT o.k FROM o WHERE o.k NOT IN (
+  SELECT decoy_hit.id FROM l_miss, decoy_hit WHERE l_miss.id = o.k
+) ORDER BY o.k;
+----
+1
+2
+3
+
+query I
+SELECT o.k FROM o WHERE NOT EXISTS (
+  SELECT decoy_hit.id FROM l_miss, decoy_hit WHERE l_miss.id = o.k
+) ORDER BY o.k;
+----
+1
+2
+3
+
+statement ok
+DROP TABLE l_miss;
+
+statement ok
+DROP TABLE decoy_hit;
+
+statement ok
+CREATE TABLE l(id INT, v INT) AS VALUES (1, 10), (2, 20);
+
+statement ok
+CREATE TABLE decoy(id INT) AS VALUES (100);
+
+query I
+SELECT o.k FROM o WHERE NOT EXISTS (
+  SELECT decoy.id, l.v FROM l, decoy WHERE l.id = o.k
+) ORDER BY o.k;
+----
+3
+
+statement ok
+DROP TABLE o;
+
+statement ok
+DROP TABLE l;
+
+statement ok
+DROP TABLE decoy;