fix(cli): correct input normalization regressions

Expressif.Cli.Tests/CliCommandTests.cs

@@ -416,6 +416,41 @@ public async Task Evaluate_ExpressionFile_WithUtf8Bom_IsSupported()
         }
     }
 
+    [TestCase("utf-16-le")]
+    [TestCase("utf-16-be")]
+    [TestCase("utf-32-le")]
+    [TestCase("utf-32-be")]
+    public async Task Evaluate_ExpressionFile_WithNonUtf8Bom_ReturnsClearError(string encodingName)
+    {
+        var encoding = encodingName switch
+        {
+            "utf-16-le" => System.Text.Encoding.Unicode,
+            "utf-16-be" => System.Text.Encoding.BigEndianUnicode,
+            "utf-32-le" => new System.Text.UTF32Encoding(bigEndian: false, byteOrderMark: true),
+            "utf-32-be" => new System.Text.UTF32Encoding(bigEndian: true, byteOrderMark: true),
+            _ => throw new ArgumentOutOfRangeException(nameof(encodingName)),
+        };
+        var path = Path.Combine(Path.GetTempPath(), $"expressif-bom-{Guid.NewGuid():N}.expr");
+        File.WriteAllText(path, "trim | upper", encoding);
+
+        try
+        {
+            var result = await InvokeAsync("evaluate", "--file", path, "--input", "nikola");
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.InvalidExpressionOrInput));
+                Assert.That(result.StdOut, Is.Empty);
+                Assert.That(result.StdErr.Trim(), Is.EqualTo($"Expression file '{path}' could not be decoded as UTF-8."));
+            });
+        }
+        finally
+        {
+            if (File.Exists(path))
+                File.Delete(path);
+        }
+    }
+
     [Test]
     public async Task Evaluate_ExpressionFile_RelativePath_IsResolvedFromCurrentDirectory()
     {
@@ -748,6 +783,22 @@ public async Task Run_BatchOption_RecordValue_ReturnsClearEnumerableError()
         });
     }
 
+    [Test]
+    public async Task Run_BatchOption_RepeatedEqualsSyntax_ReturnsDuplicateError()
+    {
+        var result = await InvokeAsync("run", "trim", "--batch=a", "--batch=b");
+
+        Assert.That(result.StdErr, Does.Contain("The --batch option can only be specified once."));
+    }
+
+    [Test]
+    public async Task Run_BatchOption_RepeatedBareTokens_DoesNotReturnDuplicateError()
+    {
+        var result = await InvokeAsync("run", "trim", "--batch", "--batch");
+
+        Assert.That(result.StdErr, Does.Not.Contain("The --batch option can only be specified once."));
+    }
+
     [Test]
     public async Task Run_SourceEnumerableExpression_EvaluatesEachRow()
     {
@@ -986,6 +1037,42 @@ public async Task Run_SourceCsv_WithExplicitHeader_UsesPocketCsvHeaderNames()
         });
     }
 
+    [Test]
+    public async Task Run_SourceCsv_WithMultipleHeaderRows_DoesNotEmitAdditionalHeaders()
+    {
+        var sourcePath = CreateTempFile($"person{Environment.NewLine}name{Environment.NewLine}Alice{Environment.NewLine}Bob", ".csv");
+
+        var result = await InvokeAsync(
+            "run", "upper", "--source", sourcePath, "--scalar",
+            "--source-option", "header-rows={1, 2}");
+
+        var outputs = result.StdOut.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.Success));
+            Assert.That(outputs, Is.EqualTo(new[] { "ALICE", "BOB" }));
+            Assert.That(result.StdErr, Is.Empty);
+        });
+    }
+
+    [Test]
+    public async Task Run_SourceCsv_WithRepeatedHeaders_DoesNotEmitRepeatedHeader()
+    {
+        var sourcePath = CreateTempFile($"name{Environment.NewLine}Alice{Environment.NewLine}name{Environment.NewLine}Bob", ".csv");
+
+        var result = await InvokeAsync(
+            "run", "upper", "--source", sourcePath, "--scalar",
+            "--source-option", "header-repeat=#true");
+
+        var outputs = result.StdOut.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.Success));
+            Assert.That(outputs, Is.EqualTo(new[] { "ALICE", "BOB" }));
+            Assert.That(result.StdErr, Is.Empty);
+        });
+    }
+
     [Test]
     public async Task Run_SourceOption_WithoutSource_ReturnsClearError()
     {

Expressif.Cli.Tests/InfrastructureCoverageTests.cs

@@ -40,6 +40,20 @@ public void Normalize_MultiColumnReaderInScalarMode_ReportsColumnCount()
             Throws.TypeOf<FormatException>().With.Message.Contains("exactly one column; found 2"));
     }
 
+    [Test]
+    public void Normalize_ReaderWithDuplicateColumnNames_RejectsDuplicate()
+    {
+        var table = new DataTable();
+        table.Columns.Add("first");
+        table.Columns.Add("second");
+        table.Rows.Add("alpha", "beta");
+        using var reader = new DuplicateNameDataReader(table.CreateDataReader());
+
+        Assert.That(
+            () => CreateInfrastructure().Normalize(reader, "source.sql").ToArray(),
+            Throws.TypeOf<FormatException>().With.Message.Contains("duplicate column name 'name'"));
+    }
+
     [Test]
     public void Normalize_CsvReaderWithEmptyHeader_ReportsFieldPosition()
     {
@@ -236,4 +250,41 @@ private sealed class TrackingDisposable : IDisposable
     {
         public void Dispose() { }
     }
+
+    private sealed class DuplicateNameDataReader(IDataReader inner) : IDataReader
+    {
+        public object this[int i] => inner[i];
+        public object this[string name] => inner[name];
+        public int Depth => inner.Depth;
+        public bool IsClosed => inner.IsClosed;
+        public int RecordsAffected => inner.RecordsAffected;
+        public int FieldCount => inner.FieldCount;
+        public void Close() => inner.Close();
+        public void Dispose() => inner.Dispose();
+        public bool GetBoolean(int i) => inner.GetBoolean(i);
+        public byte GetByte(int i) => inner.GetByte(i);
+        public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => inner.GetBytes(i, fieldOffset, buffer, bufferoffset, length);
+        public char GetChar(int i) => inner.GetChar(i);
+        public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => inner.GetChars(i, fieldoffset, buffer, bufferoffset, length);
+        public IDataReader GetData(int i) => inner.GetData(i);
+        public string GetDataTypeName(int i) => inner.GetDataTypeName(i);
+        public DateTime GetDateTime(int i) => inner.GetDateTime(i);
+        public decimal GetDecimal(int i) => inner.GetDecimal(i);
+        public double GetDouble(int i) => inner.GetDouble(i);
+        public Type GetFieldType(int i) => inner.GetFieldType(i);
+        public float GetFloat(int i) => inner.GetFloat(i);
+        public Guid GetGuid(int i) => inner.GetGuid(i);
+        public short GetInt16(int i) => inner.GetInt16(i);
+        public int GetInt32(int i) => inner.GetInt32(i);
+        public long GetInt64(int i) => inner.GetInt64(i);
+        public string GetName(int i) => "name";
+        public int GetOrdinal(string name) => inner.GetOrdinal(name);
+        public DataTable? GetSchemaTable() => inner.GetSchemaTable();
+        public string GetString(int i) => inner.GetString(i);
+        public object GetValue(int i) => inner.GetValue(i);
+        public int GetValues(object[] values) => inner.GetValues(values);
+        public bool IsDBNull(int i) => inner.IsDBNull(i);
+        public bool NextResult() => inner.NextResult();
+        public bool Read() => inner.Read();
+    }
 }

Expressif.Cli/Commands/RunCommand.cs

@@ -10,7 +10,7 @@ public static Command Create(RunHandler handler)
         var expression = new Argument<string?>("expression") { Arity = ArgumentArity.ZeroOrOne, Description = "Expression to evaluate." };
         var input = new Option<string[]>("--input") { Description = "Input row passed to the expression. Repeat --input to add rows." };
         input.Aliases.Add("-i");
-        var batch = new Option<string?>("--batch") { Description = "Enumerable batch input. Each direct element is evaluated as one row." };
+        var batch = new Option<string[]>("--batch") { Description = "Enumerable batch input. Each direct element is evaluated as one row." };
         var file = new Option<string?>("--file") { Description = "Path to a UTF-8 file containing the expression to evaluate." };
         file.Aliases.Add("-f");
         var source = new Option<string?>("--source") { Description = "Path to a source file returning rows as IEnumerable or IDataReader." };
@@ -26,10 +26,10 @@ public static Command Create(RunHandler handler)
         command.Options.Add(sourceOptions);
         command.Options.Add(file);
         command.SetAction(result => handler.Execute(new RunRequest(
-            result.GetValue(expression), result.GetValue(file), result.GetValue(input) ?? [], result.GetValue(batch),
+            result.GetValue(expression), result.GetValue(file), result.GetValue(input) ?? [], result.GetValue(batch)?.FirstOrDefault(),
             result.GetValue(source), result.GetValue(sourceOptions) ?? [], result.GetValue(scalar),
             result.GetResult(input) is not null, result.GetResult(batch) is not null, result.GetResult(source) is not null,
-            result.GetResult(sourceOptions) is not null, result.Tokens.Count(token => token.Value is "--batch"))));
+            result.GetResult(sourceOptions) is not null, result.GetResult(batch)?.IdentifierTokenCount ?? 0)));
         return command;
     }
 }

Expressif.Cli/Infrastructure/OwnedDataReader.cs

@@ -10,7 +10,8 @@ internal interface IHeaderDataReader : IDataReader
 internal sealed class OwnedDataReader(
     IDataReader inner,
     IDisposable owner,
-    bool headersAreRows = false) : IHeaderDataReader
+    bool headersAreRows = false,
+    bool skipRepeatedHeaders = false) : IHeaderDataReader
 {
     public bool HeadersAreRows { get; } = headersAreRows;
 
@@ -35,7 +36,27 @@ public void Close()
 
     public DataTable GetSchemaTable() => inner.GetSchemaTable()!;
     public bool NextResult() => inner.NextResult();
-    public bool Read() => inner.Read();
+    public bool Read()
+    {
+        while (inner.Read())
+        {
+            if (!skipRepeatedHeaders || !IsHeaderRow())
+                return true;
+        }
+
+        return false;
+    }
+
+    private bool IsHeaderRow()
+    {
+        for (var i = 0; i < FieldCount; i++)
+        {
+            if (!string.Equals(Convert.ToString(inner.GetValue(i)), inner.GetName(i), StringComparison.OrdinalIgnoreCase))
+                return false;
+        }
+
+        return true;
+    }
 
     public void Dispose()
     {

Expressif.Cli/Infrastructure/SourceInfrastructure.cs

@@ -214,9 +214,13 @@
         var fields = reader.FieldCount;
         var names = new string[fields];
         var values = new object?[fields];
+        var nameSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
         for (var i = 0; i < fields; i++)
         {
             names[i] = reader.GetName(i);
+            if (!nameSet.Add(names[i]))
+                throw new FormatException($"The source contains duplicate column name '{names[i]}'.");
+
             var value = reader.GetValue(i);
             values[i] = value is DBNull ? null : value;
         }
@@ -283,13 +287,19 @@
         {
             stream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
             var (profile, headersAreRows) = BuildCsvProfile(sourceOptions);
-            // Expressif builds dynamic row records from the CSV header. PocketCsvReader's
-            // schema-driven header mode requires a named schema, so keep header rows visible
-            // to the row adapter while retaining the configured profile for validation.
-            var readerProfile = profile.Dialect.Header ? WithoutCsvHeaderConsumption(profile) : profile;
+            var useConfiguredHeaderProcessing = sourceOptions.Any(IsHeaderProcessingOption);
+            // Expressif normally builds dynamic records from a visible first header row.
+            // Configured multi-row or repeating headers must instead be consumed by PocketCsvReader.
+            var readerProfile = useConfiguredHeaderProcessing
+                ? WithCsvHeaderConsumption(profile)
+                : profile.Dialect.Header ? WithoutCsvHeaderConsumption(profile) : profile;
             var csvReader = new CsvReader(readerProfile);
             var csvDataReader = csvReader.ToDataReader(stream);
-            return new OwnedDataReader(csvDataReader, stream, headersAreRows);
+            return new OwnedDataReader(
+                csvDataReader,
+                stream,
+                headersAreRows && !useConfiguredHeaderProcessing,
+                skipRepeatedHeaders: useConfiguredHeaderProcessing && profile.Dialect.HeaderRepeat);
         }
         catch
         {
@@ -301,6 +311,25 @@
     internal (CsvProfile Profile, bool HeadersAreRows) BuildCsvProfile(IReadOnlyList<string> options)
         => new CsvSourceProfileBuilder(values).Build(options);
 
+    private static bool IsHeaderProcessingOption(string option)
+    {
+        var separator = option.IndexOf('=');
+        var name = separator < 0 ? option : option[..separator];
+        return name.Trim() is "header-rows" or "header-repeat";
+    }
+
+    private static CsvProfile WithCsvHeaderConsumption(CsvProfile profile)
+    {
+        var dialect = profile.Dialect;
+        var headerRows = dialect.HeaderRows.Length == 0 ? new[] { 1 } : dialect.HeaderRows;
+        var readerDialect = new DialectDescriptor(
+            true, headerRows, dialect.HeaderJoin, dialect.HeaderRepeat, dialect.CommentRows, dialect.CommentChar,
+            dialect.Delimiter, dialect.LineTerminator, dialect.QuoteChar, dialect.DoubleQuote,
+            dialect.EscapeChar, dialect.NullSequence, dialect.MissingCell, dialect.SkipInitialSpace,
+            dialect.ArrayDelimiter, dialect.ArrayPrefix, dialect.ArraySuffix);
+        return new CsvProfile(readerDialect, profile.Schema, profile.Resource, profile.Parsers);
+    }
+
     private static CsvProfile WithoutCsvHeaderConsumption(CsvProfile profile)
     {
         var dialect = profile.Dialect;

Expressif.Cli/Infrastructure/StrictUtf8TextReader.cs

@@ -36,6 +36,12 @@ public string Read(string path, bool requireContent = true)
         try
         {
             using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+            Span<byte> prefix = stackalloc byte[4];
+            var prefixLength = stream.Read(prefix);
+            stream.Position = 0;
+            if (HasUnsupportedByteOrderMark(prefix[..prefixLength]))
+                throw new DecoderFallbackException("The file uses a non-UTF-8 byte order mark.");
+
             using var reader = new StreamReader(stream, new UTF8Encoding(false, true), true);
             text = reader.ReadToEnd();
         }
@@ -52,4 +58,10 @@ public string Read(string path, bool requireContent = true)
             throw new TextFileReadException(path, TextFileFailureKind.Empty);
         return text;
     }
+
+    private static bool HasUnsupportedByteOrderMark(ReadOnlySpan<byte> prefix)
+        => prefix.StartsWith(new byte[] { 0xFF, 0xFE, 0x00, 0x00 })
+            || prefix.StartsWith(new byte[] { 0x00, 0x00, 0xFE, 0xFF })
+            || prefix.StartsWith(new byte[] { 0xFF, 0xFE })
+            || prefix.StartsWith(new byte[] { 0xFE, 0xFF });
 }