Read record navigation keys without the list's display columns

modules/backend/behaviors/FormController.php

@@ -690,11 +690,11 @@ public function formRenderRecordNavigation(): string
      * Resolves the position of the current record within the controller's list
      * and the neighboring record keys used for previous/next navigation.
      *
-     * The sibling set comes from the ListController's prepared query, so it
+     * The sibling set comes from the ListController's list widget, so it
      * reflects the active filters, search and sorting exactly as the user left
-     * the list. Ordered keys are read with a single portable `pluck` and the
-     * position is resolved in PHP — no driver-specific SQL — so it behaves
-     * identically across every database Winter supports.
+     * the list. Ordered keys are read with a single portable, key-only query
+     * and the position is resolved in PHP — no driver-specific SQL — so it
+     * behaves identically across every database Winter supports.
      *
      * `recordNavigation` accepts `false` to disable navigation, or the name of a
      * list definition to navigate that list instead of the primary one, and may be
@@ -735,9 +735,7 @@ public function formGetRecordNavigation($model = null): ?array
             return null;
         }
 
-        $keys = $listWidget->prepareQuery()->pluck($model->getQualifiedKeyName())->all();
-
-        return static::resolveRecordPosition($keys, $model->getKey());
+        return static::resolveRecordPosition($listWidget->getRecordKeys(), $model->getKey());
     }
 
     /**

modules/backend/tests/widgets/ListsTest.php

@@ -2,6 +2,7 @@
 
 namespace Backend\Tests\Widgets;
 
+use Db;
 use System\Tests\Bootstrap\PluginTestCase;
 use Winter\Storm\Exception\ApplicationException;
 use Backend\Tests\Fixtures\Models\UserFixture;
@@ -120,6 +121,115 @@ public function testRestrictedColumnSinglePermissionWithUserWithRightPermissions
         $this->assertNotNull($list->getColumn('email'));
     }
 
+    /**
+     * @dataProvider keyOnlyRecordKeysProvider
+     */
+    public function testRecordKeysAreReadWithoutTheDisplayColumns(string $sortColumn, ?callable $extendQuery)
+    {
+        $this->actingAs((new UserFixture)->asSuperUser());
+
+        $sql = $this->recordKeysQuery($sortColumn, $extendQuery);
+
+        $this->assertStringStartsWith($this->keyOnlySelect(), $sql);
+        $this->assertStringNotContainsString('groups_count', $sql);
+    }
+
+    public function keyOnlyRecordKeysProvider(): array
+    {
+        return [
+            'sort by a table column' => ['id', null],
+            'where and eager load from an extension' => ['id', fn ($query) => $query->where('is_activated', true)->with('groups')],
+            'qualified order from an extension' => ['id', fn ($query) => $query->orderBy('backend_users.email')],
+        ];
+    }
+
+    /**
+     * @dataProvider fullSelectRecordKeysProvider
+     */
+    public function testRecordKeysKeepTheDisplayColumnsWhenTheQueryReliesOnThem(string $sortColumn, ?callable $extendQuery)
+    {
+        $this->actingAs((new UserFixture)->asSuperUser());
+
+        $sql = $this->recordKeysQuery($sortColumn, $extendQuery);
+
+        $this->assertStringStartsNotWith($this->keyOnlySelect(), $sql);
+        $this->assertStringContainsString('groups_count', $sql);
+    }
+
+    public function fullSelectRecordKeysProvider(): array
+    {
+        return [
+            'sort by a relation count' => ['groups', null],
+            'sort by a select alias' => ['full_name', null],
+            'having on an alias from an extension' => ['id', fn ($query) => $query->having('groups_count', '>', 0)],
+            'order by an alias from an extension' => ['id', fn ($query) => $query->addSelect(Db::raw('1 as weight'))->orderBy('weight')],
+            'raw order from an extension' => ['id', fn ($query) => $query->orderByRaw('last_name is null')],
+            'group by from an extension' => ['id', fn ($query) => $query->groupBy('backend_users.id')],
+            'distinct from an extension' => ['id', fn ($query) => $query->distinct()],
+        ];
+    }
+
+    public function testRecordKeysKeepTheDisplayColumnsWhenAGlobalScopeReliesOnThem()
+    {
+        $this->actingAs((new UserFixture)->asSuperUser());
+
+        $model = new class extends User {
+            protected static function booted()
+            {
+                static::addGlobalScope('distinct', fn ($query) => $query->distinct());
+            }
+        };
+
+        $sql = $this->recordKeysQuery('id', null, $model);
+
+        $this->assertStringStartsWith('select distinct', $sql);
+        $this->assertStringContainsString('groups_count', $sql);
+    }
+
+    /**
+     * Returns the SQL of the query Lists::getRecordKeys() runs for the given sort,
+     * with an optional `list.extendQuery` handler.
+     */
+    protected function recordKeysQuery(string $sortColumn, ?callable $extendQuery = null, ?User $model = null): string
+    {
+        $list = new Lists(null, [
+            'model' => $model ?? new User,
+            'arrayName' => 'array',
+            'defaultSort' => ['column' => $sortColumn, 'direction' => 'desc'],
+            'columns' => [
+                'id' => [
+                    'type' => 'text',
+                    'label' => 'ID',
+                    'sortable' => true
+                ],
+                'full_name' => [
+                    'label' => 'Name',
+                    'select' => 'first_name',
+                    'sortable' => true
+                ],
+                'groups' => [
+                    'label' => 'Groups',
+                    'relation' => 'groups',
+                    'useRelationCount' => true,
+                    'sortable' => true
+                ]
+            ]
+        ]);
+
+        if ($extendQuery) {
+            $list->bindEvent('list.extendQuery', $extendQuery);
+        }
+
+        $queries = Db::connection()->pretend(fn () => $list->getRecordKeys());
+
+        return end($queries)['query'];
+    }
+
+    protected function keyOnlySelect(): string
+    {
+        return 'select ' . Db::connection()->getQueryGrammar()->wrap((new User)->getQualifiedKeyName()) . ' from';
+    }
+
     protected function restrictedListsFixture(bool $singlePermission = false)
     {
         return new Lists(null, [

modules/backend/widgets/Lists.php

@@ -8,6 +8,7 @@
 use Backend\Facades\BackendAuth;
 use Backend\Traits\PreferenceMaker;
 use Carbon\Carbon;
+use Illuminate\Database\Query\Builder as QueryBuilder;
 use Illuminate\Database\QueryException;
 use Illuminate\Support\Facades\Lang;
 use System\Classes\ImageResizer;
@@ -770,6 +771,97 @@ public function prepareQuery()
         return $query;
     }
 
+    /**
+     * Returns the primary key of every record in the list, in the list's current
+     * order, honouring the active search, filters and sorting.
+     *
+     * Only the key is read: `prepareQuery()` selects every visible column, so on
+     * a list carrying `useRelationCount` or custom `select:` columns each row
+     * would evaluate a correlated subquery whose value is then thrown away. The
+     * select list is only reduced to the key when nothing in the final query -
+     * including clauses added by filters, query extensions and global scopes -
+     * can resolve against it; otherwise the query runs as built.
+     *
+     * @return array<int, mixed>
+     */
+    public function getRecordKeys(): array
+    {
+        $keyName = $this->model->getQualifiedKeyName();
+        $baseQuery = $this->prepareQuery()->toBase();
+
+        if (!$this->queryReliesOnSelectList($baseQuery)) {
+            $baseQuery->columns = [$keyName];
+
+            // The select bindings belong to the expressions just discarded.
+            $baseQuery->bindings['select'] = [];
+        }
+
+        return $baseQuery->pluck($keyName)->all();
+    }
+
+    /**
+     * Determines whether the query may refer to its select list, so that reducing
+     * it would break the query or silently change its result: any HAVING, GROUP BY,
+     * UNION or DISTINCT, a raw ORDER BY, or an ORDER BY on a selected alias.
+     */
+    protected function queryReliesOnSelectList(QueryBuilder $query): bool
+    {
+        if ($query->havings || $query->groups || $query->unions || $query->distinct) {
+            return true;
+        }
+
+        if (empty($query->orders)) {
+            return false;
+        }
+
+        $aliases = $this->getSelectAliases($query);
+
+        foreach ($query->orders as $order) {
+            $column = $order['column'] ?? null;
+
+            if (!is_string($column)) {
+                return true;
+            }
+
+            if (str_contains($column, '.')) {
+                continue;
+            }
+
+            if ($aliases === null || in_array(strtolower($column), $aliases, true)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Returns the lowercased names the query's select list exposes besides the
+     * plain `table.*` columns, or null when an expression's name can't be read.
+     *
+     * @return array<int, string>|null
+     */
+    protected function getSelectAliases(QueryBuilder $query): ?array
+    {
+        $grammar = $query->getGrammar();
+        $aliases = [];
+
+        foreach ($query->columns ?? [] as $column) {
+            $isExpression = $grammar->isExpression($column);
+            $sql = trim((string) ($isExpression ? $grammar->getValue($column) : $column));
+
+            if (preg_match('/\sas\s+[`"\[]?(\w+)[`"\]]?$/i', $sql, $match)) {
+                $aliases[] = strtolower($match[1]);
+            } elseif ($isExpression) {
+                return null;
+            } elseif (!str_ends_with($sql, '*')) {
+                $aliases[] = strtolower(Str::afterLast($sql, '.'));
+            }
+        }
+
+        return $aliases;
+    }
+
     /**
      * Returns a fingerprint of the records the list currently matches - the search term,
      * filter scopes and any query extensions, but neither the ordering nor the visible