diff --git a/Expressif.Cli.Tests/CliCommandTests.cs b/Expressif.Cli.Tests/CliCommandTests.cs index 8bc4e8d6..e49e206f 100644 --- a/Expressif.Cli.Tests/CliCommandTests.cs +++ b/Expressif.Cli.Tests/CliCommandTests.cs @@ -416,6 +416,41 @@ public async Task Evaluate_ExpressionFile_WithUtf8Bom_IsSupported() } } + [TestCase("utf-16-le")] + [TestCase("utf-16-be")] + [TestCase("utf-32-le")] + [TestCase("utf-32-be")] + public async Task Evaluate_ExpressionFile_WithNonUtf8Bom_ReturnsClearError(string encodingName) + { + var encoding = encodingName switch + { + "utf-16-le" => System.Text.Encoding.Unicode, + "utf-16-be" => System.Text.Encoding.BigEndianUnicode, + "utf-32-le" => new System.Text.UTF32Encoding(bigEndian: false, byteOrderMark: true), + "utf-32-be" => new System.Text.UTF32Encoding(bigEndian: true, byteOrderMark: true), + _ => throw new ArgumentOutOfRangeException(nameof(encodingName)), + }; + var path = Path.Combine(Path.GetTempPath(), $"expressif-bom-{Guid.NewGuid():N}.expr"); + File.WriteAllText(path, "trim | upper", encoding); + + try + { + var result = await InvokeAsync("evaluate", "--file", path, "--input", "nikola"); + + Assert.Multiple(() => + { + Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.InvalidExpressionOrInput)); + Assert.That(result.StdOut, Is.Empty); + Assert.That(result.StdErr.Trim(), Is.EqualTo($"Expression file '{path}' could not be decoded as UTF-8.")); + }); + } + finally + { + if (File.Exists(path)) + File.Delete(path); + } + } + [Test] public async Task Evaluate_ExpressionFile_RelativePath_IsResolvedFromCurrentDirectory() { @@ -748,6 +783,22 @@ public async Task Run_BatchOption_RecordValue_ReturnsClearEnumerableError() }); } + [Test] + public async Task Run_BatchOption_RepeatedEqualsSyntax_ReturnsDuplicateError() + { + var result = await InvokeAsync("run", "trim", "--batch=a", "--batch=b"); + + Assert.That(result.StdErr, Does.Contain("The --batch option can only be specified once.")); + } + + [Test] + public async Task Run_BatchOption_RepeatedBareTokens_DoesNotReturnDuplicateError() + { + var result = await InvokeAsync("run", "trim", "--batch", "--batch"); + + Assert.That(result.StdErr, Does.Not.Contain("The --batch option can only be specified once.")); + } + [Test] public async Task Run_SourceEnumerableExpression_EvaluatesEachRow() { @@ -986,6 +1037,42 @@ public async Task Run_SourceCsv_WithExplicitHeader_UsesPocketCsvHeaderNames() }); } + [Test] + public async Task Run_SourceCsv_WithMultipleHeaderRows_DoesNotEmitAdditionalHeaders() + { + var sourcePath = CreateTempFile($"person{Environment.NewLine}name{Environment.NewLine}Alice{Environment.NewLine}Bob", ".csv"); + + var result = await InvokeAsync( + "run", "upper", "--source", sourcePath, "--scalar", + "--source-option", "header-rows={1, 2}"); + + var outputs = result.StdOut.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + Assert.Multiple(() => + { + Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.Success)); + Assert.That(outputs, Is.EqualTo(new[] { "ALICE", "BOB" })); + Assert.That(result.StdErr, Is.Empty); + }); + } + + [Test] + public async Task Run_SourceCsv_WithRepeatedHeaders_DoesNotEmitRepeatedHeader() + { + var sourcePath = CreateTempFile($"name{Environment.NewLine}Alice{Environment.NewLine}name{Environment.NewLine}Bob", ".csv"); + + var result = await InvokeAsync( + "run", "upper", "--source", sourcePath, "--scalar", + "--source-option", "header-repeat=#true"); + + var outputs = result.StdOut.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + Assert.Multiple(() => + { + Assert.That(result.ExitCode, Is.EqualTo(ExitCodes.Success)); + Assert.That(outputs, Is.EqualTo(new[] { "ALICE", "BOB" })); + Assert.That(result.StdErr, Is.Empty); + }); + } + [Test] public async Task Run_SourceOption_WithoutSource_ReturnsClearError() { diff --git a/Expressif.Cli.Tests/InfrastructureCoverageTests.cs b/Expressif.Cli.Tests/InfrastructureCoverageTests.cs index cb9ab0bd..0ec29708 100644 --- a/Expressif.Cli.Tests/InfrastructureCoverageTests.cs +++ b/Expressif.Cli.Tests/InfrastructureCoverageTests.cs @@ -40,6 +40,20 @@ public void Normalize_MultiColumnReaderInScalarMode_ReportsColumnCount() Throws.TypeOf().With.Message.Contains("exactly one column; found 2")); } + [Test] + public void Normalize_ReaderWithDuplicateColumnNames_RejectsDuplicate() + { + var table = new DataTable(); + table.Columns.Add("first"); + table.Columns.Add("second"); + table.Rows.Add("alpha", "beta"); + using var reader = new DuplicateNameDataReader(table.CreateDataReader()); + + Assert.That( + () => CreateInfrastructure().Normalize(reader, "source.sql").ToArray(), + Throws.TypeOf().With.Message.Contains("duplicate column name 'name'")); + } + [Test] public void Normalize_CsvReaderWithEmptyHeader_ReportsFieldPosition() { @@ -236,4 +250,41 @@ private sealed class TrackingDisposable : IDisposable { public void Dispose() { } } + + private sealed class DuplicateNameDataReader(IDataReader inner) : IDataReader + { + public object this[int i] => inner[i]; + public object this[string name] => inner[name]; + public int Depth => inner.Depth; + public bool IsClosed => inner.IsClosed; + public int RecordsAffected => inner.RecordsAffected; + public int FieldCount => inner.FieldCount; + public void Close() => inner.Close(); + public void Dispose() => inner.Dispose(); + public bool GetBoolean(int i) => inner.GetBoolean(i); + public byte GetByte(int i) => inner.GetByte(i); + public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => inner.GetBytes(i, fieldOffset, buffer, bufferoffset, length); + public char GetChar(int i) => inner.GetChar(i); + public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => inner.GetChars(i, fieldoffset, buffer, bufferoffset, length); + public IDataReader GetData(int i) => inner.GetData(i); + public string GetDataTypeName(int i) => inner.GetDataTypeName(i); + public DateTime GetDateTime(int i) => inner.GetDateTime(i); + public decimal GetDecimal(int i) => inner.GetDecimal(i); + public double GetDouble(int i) => inner.GetDouble(i); + public Type GetFieldType(int i) => inner.GetFieldType(i); + public float GetFloat(int i) => inner.GetFloat(i); + public Guid GetGuid(int i) => inner.GetGuid(i); + public short GetInt16(int i) => inner.GetInt16(i); + public int GetInt32(int i) => inner.GetInt32(i); + public long GetInt64(int i) => inner.GetInt64(i); + public string GetName(int i) => "name"; + public int GetOrdinal(string name) => inner.GetOrdinal(name); + public DataTable? GetSchemaTable() => inner.GetSchemaTable(); + public string GetString(int i) => inner.GetString(i); + public object GetValue(int i) => inner.GetValue(i); + public int GetValues(object[] values) => inner.GetValues(values); + public bool IsDBNull(int i) => inner.IsDBNull(i); + public bool NextResult() => inner.NextResult(); + public bool Read() => inner.Read(); + } } diff --git a/Expressif.Cli/Commands/RunCommand.cs b/Expressif.Cli/Commands/RunCommand.cs index 30e89f81..7ddb65c3 100644 --- a/Expressif.Cli/Commands/RunCommand.cs +++ b/Expressif.Cli/Commands/RunCommand.cs @@ -10,7 +10,7 @@ public static Command Create(RunHandler handler) var expression = new Argument("expression") { Arity = ArgumentArity.ZeroOrOne, Description = "Expression to evaluate." }; var input = new Option("--input") { Description = "Input row passed to the expression. Repeat --input to add rows." }; input.Aliases.Add("-i"); - var batch = new Option("--batch") { Description = "Enumerable batch input. Each direct element is evaluated as one row." }; + var batch = new Option("--batch") { Description = "Enumerable batch input. Each direct element is evaluated as one row." }; var file = new Option("--file") { Description = "Path to a UTF-8 file containing the expression to evaluate." }; file.Aliases.Add("-f"); var source = new Option("--source") { Description = "Path to a source file returning rows as IEnumerable or IDataReader." }; @@ -26,10 +26,10 @@ public static Command Create(RunHandler handler) command.Options.Add(sourceOptions); command.Options.Add(file); command.SetAction(result => handler.Execute(new RunRequest( - result.GetValue(expression), result.GetValue(file), result.GetValue(input) ?? [], result.GetValue(batch), + result.GetValue(expression), result.GetValue(file), result.GetValue(input) ?? [], result.GetValue(batch)?.FirstOrDefault(), result.GetValue(source), result.GetValue(sourceOptions) ?? [], result.GetValue(scalar), result.GetResult(input) is not null, result.GetResult(batch) is not null, result.GetResult(source) is not null, - result.GetResult(sourceOptions) is not null, result.Tokens.Count(token => token.Value is "--batch")))); + result.GetResult(sourceOptions) is not null, result.GetResult(batch)?.IdentifierTokenCount ?? 0))); return command; } } diff --git a/Expressif.Cli/Infrastructure/OwnedDataReader.cs b/Expressif.Cli/Infrastructure/OwnedDataReader.cs index 492ed9ba..f1e1e3a4 100644 --- a/Expressif.Cli/Infrastructure/OwnedDataReader.cs +++ b/Expressif.Cli/Infrastructure/OwnedDataReader.cs @@ -10,7 +10,8 @@ internal interface IHeaderDataReader : IDataReader internal sealed class OwnedDataReader( IDataReader inner, IDisposable owner, - bool headersAreRows = false) : IHeaderDataReader + bool headersAreRows = false, + bool skipRepeatedHeaders = false) : IHeaderDataReader { public bool HeadersAreRows { get; } = headersAreRows; @@ -35,7 +36,27 @@ public void Close() public DataTable GetSchemaTable() => inner.GetSchemaTable()!; public bool NextResult() => inner.NextResult(); - public bool Read() => inner.Read(); + public bool Read() + { + while (inner.Read()) + { + if (!skipRepeatedHeaders || !IsHeaderRow()) + return true; + } + + return false; + } + + private bool IsHeaderRow() + { + for (var i = 0; i < FieldCount; i++) + { + if (!string.Equals(Convert.ToString(inner.GetValue(i)), inner.GetName(i), StringComparison.OrdinalIgnoreCase)) + return false; + } + + return true; + } public void Dispose() { diff --git a/Expressif.Cli/Infrastructure/SourceInfrastructure.cs b/Expressif.Cli/Infrastructure/SourceInfrastructure.cs index 7525a292..463cacb6 100644 --- a/Expressif.Cli/Infrastructure/SourceInfrastructure.cs +++ b/Expressif.Cli/Infrastructure/SourceInfrastructure.cs @@ -214,9 +214,13 @@ private static RecordValue BuildRecordValue(IDataReader reader) var fields = reader.FieldCount; var names = new string[fields]; var values = new object?[fields]; + var nameSet = new HashSet(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < fields; i++) { names[i] = reader.GetName(i); + if (!nameSet.Add(names[i])) + throw new FormatException($"The source contains duplicate column name '{names[i]}'."); + var value = reader.GetValue(i); values[i] = value is DBNull ? null : value; } @@ -283,13 +287,19 @@ internal IDataReader OpenCsvDataReader(string sourcePath, IReadOnlyList { stream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read); var (profile, headersAreRows) = BuildCsvProfile(sourceOptions); - // Expressif builds dynamic row records from the CSV header. PocketCsvReader's - // schema-driven header mode requires a named schema, so keep header rows visible - // to the row adapter while retaining the configured profile for validation. - var readerProfile = profile.Dialect.Header ? WithoutCsvHeaderConsumption(profile) : profile; + var useConfiguredHeaderProcessing = sourceOptions.Any(IsHeaderProcessingOption); + // Expressif normally builds dynamic records from a visible first header row. + // Configured multi-row or repeating headers must instead be consumed by PocketCsvReader. + var readerProfile = useConfiguredHeaderProcessing + ? WithCsvHeaderConsumption(profile) + : profile.Dialect.Header ? WithoutCsvHeaderConsumption(profile) : profile; var csvReader = new CsvReader(readerProfile); var csvDataReader = csvReader.ToDataReader(stream); - return new OwnedDataReader(csvDataReader, stream, headersAreRows); + return new OwnedDataReader( + csvDataReader, + stream, + headersAreRows && !useConfiguredHeaderProcessing, + skipRepeatedHeaders: useConfiguredHeaderProcessing && profile.Dialect.HeaderRepeat); } catch { @@ -301,6 +311,25 @@ internal IDataReader OpenCsvDataReader(string sourcePath, IReadOnlyList internal (CsvProfile Profile, bool HeadersAreRows) BuildCsvProfile(IReadOnlyList options) => new CsvSourceProfileBuilder(values).Build(options); + private static bool IsHeaderProcessingOption(string option) + { + var separator = option.IndexOf('='); + var name = separator < 0 ? option : option[..separator]; + return name.Trim() is "header-rows" or "header-repeat"; + } + + private static CsvProfile WithCsvHeaderConsumption(CsvProfile profile) + { + var dialect = profile.Dialect; + var headerRows = dialect.HeaderRows.Length == 0 ? new[] { 1 } : dialect.HeaderRows; + var readerDialect = new DialectDescriptor( + true, headerRows, dialect.HeaderJoin, dialect.HeaderRepeat, dialect.CommentRows, dialect.CommentChar, + dialect.Delimiter, dialect.LineTerminator, dialect.QuoteChar, dialect.DoubleQuote, + dialect.EscapeChar, dialect.NullSequence, dialect.MissingCell, dialect.SkipInitialSpace, + dialect.ArrayDelimiter, dialect.ArrayPrefix, dialect.ArraySuffix); + return new CsvProfile(readerDialect, profile.Schema, profile.Resource, profile.Parsers); + } + private static CsvProfile WithoutCsvHeaderConsumption(CsvProfile profile) { var dialect = profile.Dialect; diff --git a/Expressif.Cli/Infrastructure/StrictUtf8TextReader.cs b/Expressif.Cli/Infrastructure/StrictUtf8TextReader.cs index 639db9d9..5bc1d38d 100644 --- a/Expressif.Cli/Infrastructure/StrictUtf8TextReader.cs +++ b/Expressif.Cli/Infrastructure/StrictUtf8TextReader.cs @@ -36,6 +36,12 @@ public string Read(string path, bool requireContent = true) try { using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + Span prefix = stackalloc byte[4]; + var prefixLength = stream.Read(prefix); + stream.Position = 0; + if (HasUnsupportedByteOrderMark(prefix[..prefixLength])) + throw new DecoderFallbackException("The file uses a non-UTF-8 byte order mark."); + using var reader = new StreamReader(stream, new UTF8Encoding(false, true), true); text = reader.ReadToEnd(); } @@ -52,4 +58,10 @@ public string Read(string path, bool requireContent = true) throw new TextFileReadException(path, TextFileFailureKind.Empty); return text; } + + private static bool HasUnsupportedByteOrderMark(ReadOnlySpan prefix) + => prefix.StartsWith(new byte[] { 0xFF, 0xFE, 0x00, 0x00 }) + || prefix.StartsWith(new byte[] { 0x00, 0x00, 0xFE, 0xFF }) + || prefix.StartsWith(new byte[] { 0xFF, 0xFE }) + || prefix.StartsWith(new byte[] { 0xFE, 0xFF }); }