From 7593096f8310319561b3ea732b5e7fca1b9411f1 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sat, 22 Aug 2026 13:28:57 +0800 Subject: [PATCH] Add bounded strict portable record decoding Add schema dispatch and bounded exact JSON decoding for every v1 record family, rejecting duplicate members, unknown and case-variant fields, missing required fields, JSON numbers and nulls, invalid UTF-8, unpaired surrogates, trailing tokens, and payloads past the file, depth, member, and string limits. Add the canonical value rules the model depends on: record IDs and references, positive decimals, relative and absolute record paths, credential-free HTTPS source URLs, reversible percent-encoded tool version segments, and bounded sorted unique collections. Decoding stays free of record business rules, cross-record validation, graph traversal, and network access. Record-local validation and external-evidence validation are wired into decoding by the slices that own them, so decodeRecordV1 and decodeValidationEvidenceV1 do not call them here. Delivers PTD-03 of docs/PORTABLE_TOOL_DEFINITION_IMPLEMENTATION_PLAN.md. --- internal/toolcatalog/records_decode.go | 744 ++++++++++++++++++++ internal/toolcatalog/records_decode_test.go | 421 +++++++++++ 2 files changed, 1165 insertions(+) create mode 100644 internal/toolcatalog/records_decode.go create mode 100644 internal/toolcatalog/records_decode_test.go diff --git a/internal/toolcatalog/records_decode.go b/internal/toolcatalog/records_decode.go new file mode 100644 index 00000000..4af717e4 --- /dev/null +++ b/internal/toolcatalog/records_decode.go @@ -0,0 +1,744 @@ +package toolcatalog + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/netip" + "net/url" + "path" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/omry/reploy/internal/canonical" +) + +const ( + maxDefinitionFileBytes = 1 << 20 + maxDefinitionJSONDepth = 32 + maxDefinitionJSONMembers = 4096 + maxDefinitionJSONStringBytes = 64 << 10 + maxDefinitionReferences = 1024 +) + +var canonicalDecimalPattern = regexp.MustCompile(`^(0|[1-9][0-9]*)$`) + +type recordHeaderV1 struct { + Schema string `json:"schema"` + ID string `json:"id"` +} + +type loadedRecordV1 struct { + ID string + Schema string + Digest canonical.Digest + Path string + Value any +} + +func decodeRecordV1(filename string, payload []byte) (loadedRecordV1, error) { + if err := validateStrictJSONV1(payload); err != nil { + return loadedRecordV1{}, fmt.Errorf("decode %s: %w", filename, err) + } + var header recordHeaderV1 + if err := json.Unmarshal(payload, &header); err != nil { + return loadedRecordV1{}, fmt.Errorf("decode %s header: %w", filename, err) + } + if header.Schema == "" { + return loadedRecordV1{}, fmt.Errorf("decode %s: record schema is required", filename) + } + if header.ID == "" { + return loadedRecordV1{}, fmt.Errorf("decode %s: record ID is required", filename) + } + var value any + switch header.Schema { + case ToolRecordSchemaV1: + value = &ToolRecordV1{} + case ReleaseManifestSchemaV1: + value = &ReleaseManifestV1{} + case ReleaseContractSchemaV1: + value = &ReleaseContractV1{} + case TargetRecordSchemaV1: + value = &TargetRecordV1{} + case BindingContractSchemaV1: + value = &BindingContractV1{} + case BindingArtifactSchemaV1: + value = &BindingArtifactRecordV1{} + case PayloadRecordSchemaV1: + value = &PayloadRecordV1{} + case ArtifactSourceRecordSchemaV1: + value = &ArtifactSourceRecordV1{} + case NativePackageSetSchemaV1: + value = &NativePackageSetV1{} + case IntegrationFixtureSchemaV1: + value = &IntegrationFixtureRecordV1{} + case ValidationProfileSchemaV1: + value = &ValidationProfileRecordV1{} + default: + return loadedRecordV1{}, fmt.Errorf("decode %s: unsupported schema %q", filename, header.Schema) + } + if err := decodeExactJSONV1(payload, value); err != nil { + return loadedRecordV1{}, fmt.Errorf("decode %s: %w", filename, err) + } + record := loadedRecordV1{ID: header.ID, Schema: header.Schema, Path: filename, Value: value} + digest, err := canonical.Sum("portable-tool-record", portableToolRecordIdentityV1, value) + if err != nil { + return loadedRecordV1{}, fmt.Errorf("digest %s: %w", filename, err) + } + record.Digest = digest + return record, nil +} + +func decodeValidationEvidenceV1(filename string, payload []byte) (ValidationEvidenceV1, error) { + if err := validateStrictJSONV1(payload); err != nil { + return ValidationEvidenceV1{}, fmt.Errorf("decode %s: %w", filename, err) + } + var evidence ValidationEvidenceV1 + if err := decodeExactJSONV1(payload, &evidence); err != nil { + return ValidationEvidenceV1{}, fmt.Errorf("decode %s: %w", filename, err) + } + return evidence, nil +} + +func decodeExactJSONV1(payload []byte, target any) error { + targetType := reflect.TypeOf(target) + for targetType.Kind() == reflect.Pointer { + targetType = targetType.Elem() + } + if err := validateExactJSONMembersV1(payload, targetType); err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return fmt.Errorf("trailing JSON value") + } + return err + } + return nil +} + +func validateExactJSONMembersV1(payload json.RawMessage, target reflect.Type) error { + if bytes.Equal(bytes.TrimSpace(payload), []byte("null")) { + if target.Kind() == reflect.Pointer { + return nil + } + return fmt.Errorf("JSON null is not valid for %s", target) + } + for target.Kind() == reflect.Pointer { + target = target.Elem() + } + switch target.Kind() { + case reflect.Struct: + var members map[string]json.RawMessage + if err := json.Unmarshal(payload, &members); err != nil { + return nil + } + fields := make(map[string]reflect.Type, target.NumField()) + requiredFields := make([]string, 0, target.NumField()) + for index := 0; index < target.NumField(); index++ { + field := target.Field(index) + if !field.IsExported() { + continue + } + tag := strings.Split(field.Tag.Get("json"), ",") + name := tag[0] + if name == "-" { + continue + } + if name == "" { + name = field.Name + } + fields[name] = field.Type + if !containsRecordValueV1(tag[1:], "omitempty") { + requiredFields = append(requiredFields, name) + } + } + names := make([]string, 0, len(members)) + for name := range members { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + value := members[name] + fieldType, exists := fields[name] + if !exists { + return fmt.Errorf("unknown field %q", name) + } + if err := validateExactJSONMembersV1(value, fieldType); err != nil { + return err + } + } + for _, name := range requiredFields { + if _, exists := members[name]; !exists { + return fmt.Errorf("required field %q is missing", name) + } + } + case reflect.Slice, reflect.Array: + var elements []json.RawMessage + if err := json.Unmarshal(payload, &elements); err != nil { + return nil + } + for _, element := range elements { + if err := validateExactJSONMembersV1(element, target.Elem()); err != nil { + return err + } + } + case reflect.Map: + var members map[string]json.RawMessage + if err := json.Unmarshal(payload, &members); err != nil { + return nil + } + names := make([]string, 0, len(members)) + for name := range members { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + value := members[name] + if err := validateExactJSONMembersV1(value, target.Elem()); err != nil { + return err + } + } + } + return nil +} + +func validateStrictJSONV1(payload []byte) error { + if len(payload) == 0 || len(payload) > maxDefinitionFileBytes { + return fmt.Errorf("record size must be between 1 and %d bytes", maxDefinitionFileBytes) + } + if !utf8.Valid(payload) { + return fmt.Errorf("record must be valid UTF-8") + } + if err := validateJSONStringSurrogatesV1(payload); err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.UseNumber() + members := 0 + if err := scanStrictJSONValueV1(decoder, 0, &members); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF { + if err == nil { + return fmt.Errorf("trailing JSON token %v", token) + } + return err + } + return nil +} + +func validateJSONStringSurrogatesV1(payload []byte) error { + inString := false + for index := 0; index < len(payload); index++ { + if !inString { + if payload[index] == '"' { + inString = true + } + continue + } + switch payload[index] { + case '"': + inString = false + case '\\': + if index+1 >= len(payload) { + return fmt.Errorf("invalid JSON string escape") + } + if payload[index+1] != 'u' { + index++ + continue + } + value, ok := parseJSONHexQuadV1(payload, index+2) + if !ok { + return fmt.Errorf("invalid JSON Unicode escape") + } + if value >= 0xdc00 && value <= 0xdfff { + return fmt.Errorf("JSON string contains an unpaired UTF-16 surrogate escape") + } + if value >= 0xd800 && value <= 0xdbff { + if index+12 > len(payload) || payload[index+6] != '\\' || payload[index+7] != 'u' { + return fmt.Errorf("JSON string contains an unpaired UTF-16 surrogate escape") + } + low, validLow := parseJSONHexQuadV1(payload, index+8) + if !validLow || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("JSON string contains an unpaired UTF-16 surrogate escape") + } + index += 11 + continue + } + index += 5 + } + } + return nil +} + +func parseJSONHexQuadV1(payload []byte, start int) (uint16, bool) { + if start+4 > len(payload) { + return 0, false + } + parsed, err := strconv.ParseUint(string(payload[start:start+4]), 16, 16) + return uint16(parsed), err == nil +} + +func scanStrictJSONValueV1(decoder *json.Decoder, depth int, members *int) error { + if depth > maxDefinitionJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d", maxDefinitionJSONDepth) + } + token, err := decoder.Token() + if err != nil { + return err + } + switch value := token.(type) { + case json.Delim: + if depth >= maxDefinitionJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d", maxDefinitionJSONDepth) + } + switch value { + case '{': + seen := map[string]bool{} + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok { + return fmt.Errorf("object member name is not a string") + } + if seen[name] { + return fmt.Errorf("duplicate object member %q", name) + } + if len(name) > maxDefinitionJSONStringBytes { + return fmt.Errorf("object member name exceeds %d bytes", maxDefinitionJSONStringBytes) + } + seen[name] = true + (*members)++ + if *members > maxDefinitionJSONMembers { + return fmt.Errorf("JSON member count exceeds %d", maxDefinitionJSONMembers) + } + if err := scanStrictJSONValueV1(decoder, depth+1, members); err != nil { + return err + } + } + _, err := decoder.Token() + return err + case '[': + for decoder.More() { + (*members)++ + if *members > maxDefinitionJSONMembers { + return fmt.Errorf("JSON member count exceeds %d", maxDefinitionJSONMembers) + } + if err := scanStrictJSONValueV1(decoder, depth+1, members); err != nil { + return err + } + } + _, err := decoder.Token() + return err + default: + return fmt.Errorf("unexpected JSON delimiter %q", value) + } + case string: + if len(value) > maxDefinitionJSONStringBytes { + return fmt.Errorf("JSON string exceeds %d bytes", maxDefinitionJSONStringBytes) + } + case json.Number: + return fmt.Errorf("JSON numbers are not supported; encode schema quantities as decimal strings") + case bool, nil: + return nil + default: + return fmt.Errorf("unsupported JSON token %T", token) + } + return nil +} + +func validateRecordReferenceV1(reference RecordReferenceV1) error { + if err := validateRecordIDV1(reference.ID); err != nil { + return err + } + if err := reference.Digest.Validate(); err != nil { + return fmt.Errorf("reference %q digest: %w", reference.ID, err) + } + return nil +} + +func validateRecordIDV1(value string) error { + if value == "" || strings.TrimSpace(value) != value || !strings.HasPrefix(value, "tool:") { + return fmt.Errorf("record ID %q must be a canonical tool-qualified ID", value) + } + for index := 0; index < len(value); index++ { + character := value[index] + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' { + continue + } + switch character { + case '.', '+', '-', '_', ':', '/': + case '%': + if index+2 >= len(value) { + return fmt.Errorf("record ID %q contains an incomplete percent escape", value) + } + if _, ok := uppercaseHexValueV1(value[index+1]); !ok { + return fmt.Errorf("record ID %q percent escapes must use uppercase hexadecimal", value) + } + if _, ok := uppercaseHexValueV1(value[index+2]); !ok { + return fmt.Errorf("record ID %q percent escapes must use uppercase hexadecimal", value) + } + index += 2 + default: + return fmt.Errorf("record ID %q contains unsupported character %q", value, character) + } + } + segments := strings.Split(value, "/") + toolName := strings.TrimPrefix(segments[0], "tool:") + if !validRecordIdentifierV1(toolName) { + return fmt.Errorf("record ID %q has an invalid tool name", value) + } + for index, segment := range segments { + if segment == "" || segment == "." || segment == ".." { + return fmt.Errorf("record ID %q contains an invalid path segment", value) + } + if index != 2 && strings.Contains(segment, "%") { + return fmt.Errorf("record ID %q contains an escape outside its version segment", value) + } + } + if len(segments) == 1 { + return nil + } + if len(segments) < 4 || segments[1] != "releases" { + return fmt.Errorf("record ID %q must use a tool release namespace", value) + } + if _, err := decodeToolVersionSegmentV1(segments[2]); err != nil { + return fmt.Errorf("record ID %q version segment: %w", value, err) + } + return nil +} + +func validRecordIdentifierV1(value string) bool { + if value == "" || value[0] < 'a' || value[0] > 'z' { + return false + } + for _, character := range value[1:] { + if character < 'a' || character > 'z' { + if character < '0' || character > '9' { + if character != '-' { + return false + } + } + } + } + return true +} + +func validateCanonicalDecimalV1(field string, value string, positive bool) error { + if !canonicalDecimalPattern.MatchString(value) { + return fmt.Errorf("%s must be a canonical decimal string", field) + } + parsed, err := strconv.ParseUint(value, 10, 63) + if err != nil || positive && parsed == 0 { + return fmt.Errorf("%s must be a bounded positive decimal string", field) + } + return nil +} + +func validateReferenceListV1(field string, references []RecordReferenceV1) error { + if references == nil || len(references) > maxDefinitionReferences { + return fmt.Errorf("%s must use an array with at most %d entries", field, maxDefinitionReferences) + } + for index, reference := range references { + if err := validateRecordReferenceV1(reference); err != nil { + return fmt.Errorf("%s[%d]: %w", field, index, err) + } + if index > 0 && references[index-1].ID >= reference.ID { + return fmt.Errorf("%s must be unique and sorted by ID", field) + } + } + return nil +} + +func validateSourceURLV1(raw string) error { + canonicalURL, err := canonicalSourceURLV1(raw) + if err != nil { + return err + } + if raw != canonicalURL { + return fmt.Errorf("source URL must use canonical spelling %q", canonicalURL) + } + return nil +} + +func canonicalSourceURLV1(raw string) (string, error) { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "https" || parsed.Opaque != "" || parsed.Host == "" || parsed.Hostname() == "" || parsed.User != nil || parsed.ForceQuery || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Contains(raw, "#") || parsed.Host != strings.ToLower(parsed.Host) || strings.HasSuffix(parsed.Hostname(), ".") || parsed.Port() == "443" || !asciiURLHostV1(parsed.Hostname()) || !canonicalPercentEscapesV1(parsed.EscapedPath()) || hasURLDotSegmentV1(parsed.Path) { + return "", fmt.Errorf("source URL must be a canonical credential-free HTTPS URL without query or fragment") + } + port := parsed.Port() + if strings.HasSuffix(parsed.Host, ":") || port != "" && (!canonicalDecimalPattern.MatchString(port) || port == "0") { + return "", fmt.Errorf("source URL must use a canonical authority") + } + if port != "" { + parsedPort, err := strconv.ParseUint(port, 10, 16) + if err != nil || parsedPort == 0 { + return "", fmt.Errorf("source URL must use a canonical authority") + } + } + host := parsed.Hostname() + if address, err := netip.ParseAddr(host); err == nil { + if address.Zone() != "" { + return "", fmt.Errorf("source URL must use a canonical authority") + } + host = address.String() + if address.Is6() { + host = "[" + host + "]" + } + } else if strings.Contains(host, ":") || numericURLHostV1(host) { + return "", fmt.Errorf("source URL must use a canonical authority") + } + if port != "" { + host += ":" + port + } + escapedPath := parsed.EscapedPath() + if escapedPath == "" { + escapedPath = "/" + } + return "https://" + host + canonicalSourcePathV1(escapedPath), nil +} + +func numericURLHostV1(host string) bool { + if host == "" { + return false + } + for _, component := range strings.Split(host, ".") { + if component == "" { + return false + } + decimal := true + for _, character := range component { + if character < '0' || character > '9' { + decimal = false + break + } + } + if decimal { + continue + } + if len(component) <= 2 || !strings.HasPrefix(component, "0x") { + return false + } + for _, character := range component[2:] { + if character < '0' || character > '9' && (character < 'a' || character > 'f') { + return false + } + } + } + return true +} + +func canonicalSourcePathV1(escapedPath string) string { + var normalized strings.Builder + normalized.Grow(len(escapedPath)) + for index := 0; index < len(escapedPath); index++ { + if escapedPath[index] != '%' { + normalized.WriteByte(escapedPath[index]) + continue + } + value, err := strconv.ParseUint(escapedPath[index+1:index+3], 16, 8) + if err != nil { + normalized.WriteString(escapedPath[index:]) + break + } + character := byte(value) + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || strings.ContainsRune("-._~", rune(character)) { + normalized.WriteByte(character) + } else { + normalized.WriteString(escapedPath[index : index+3]) + } + index += 2 + } + return normalized.String() +} + +func asciiURLHostV1(host string) bool { + for _, character := range host { + if character > unicode.MaxASCII || unicode.IsControl(character) { + return false + } + } + return true +} + +func canonicalPercentEscapesV1(value string) bool { + for index := 0; index < len(value); index++ { + if value[index] != '%' { + continue + } + if index+2 >= len(value) || !uppercaseHexV1(value[index+1]) || !uppercaseHexV1(value[index+2]) { + return false + } + index += 2 + } + return true +} + +func uppercaseHexV1(value byte) bool { + return value >= '0' && value <= '9' || value >= 'A' && value <= 'F' +} + +func hasURLDotSegmentV1(value string) bool { + for _, segment := range strings.Split(value, "/") { + if segment == "." || segment == ".." { + return true + } + } + return false +} + +func validateSortedUniqueStringsV1(field string, values []string, allowEmpty bool) error { + if values == nil { + return fmt.Errorf("%s must use an array", field) + } + for index, value := range values { + if !allowEmpty && value == "" || strings.TrimSpace(value) != value || containsControlV1(value) || index > 0 && values[index-1] >= value { + return fmt.Errorf("%s must contain unique sorted canonical values", field) + } + } + return nil +} + +func validateRecordPathV1(value string, allowDot bool) error { + if value == "." && allowDot { + return nil + } + if value == "" || containsControlV1(value) || path.IsAbs(value) || path.Clean(value) != value || + strings.Contains(value, `\`) { + return fmt.Errorf("path %q must be a canonical relative slash path", value) + } + for _, segment := range strings.Split(value, "/") { + if segment == "" || segment == "." || segment == ".." { + return fmt.Errorf("path %q contains an invalid segment", value) + } + } + return nil +} + +func validateAbsoluteRecordPathV1(value string) error { + if value == "" || containsControlV1(value) || !path.IsAbs(value) || path.Clean(value) != value || value == "/" || + strings.Contains(value, `\`) { + return fmt.Errorf("path %q must be a canonical absolute non-root slash path", value) + } + return nil +} + +func encodeToolVersionSegmentV1(value string) (string, error) { + if !validRecordTokenV1(value) || !utf8.ValidString(value) { + return "", fmt.Errorf("tool version must be canonical UTF-8 text") + } + encodeDots := value == "." || value == ".." + const hex = "0123456789ABCDEF" + var encoded strings.Builder + encoded.Grow(len(value)) + for _, character := range []byte(value) { + literal := character >= 'a' && character <= 'z' || + character >= 'A' && character <= 'Z' || + character >= '0' && character <= '9' || + strings.ContainsRune(".+-_", rune(character)) + if literal && !(encodeDots && character == '.') { + encoded.WriteByte(character) + continue + } + encoded.WriteByte('%') + encoded.WriteByte(hex[character>>4]) + encoded.WriteByte(hex[character&0x0f]) + } + return encoded.String(), nil +} + +func decodeToolVersionSegmentV1(value string) (string, error) { + if value == "" { + return "", fmt.Errorf("encoded tool version must not be empty") + } + decoded := make([]byte, 0, len(value)) + for index := 0; index < len(value); index++ { + if value[index] != '%' { + decoded = append(decoded, value[index]) + continue + } + if index+2 >= len(value) { + return "", fmt.Errorf("encoded tool version contains an incomplete escape") + } + high, highOK := uppercaseHexValueV1(value[index+1]) + low, lowOK := uppercaseHexValueV1(value[index+2]) + if !highOK || !lowOK { + return "", fmt.Errorf("encoded tool version escapes must use uppercase hexadecimal") + } + decoded = append(decoded, high<<4|low) + index += 2 + } + version := string(decoded) + canonical, err := encodeToolVersionSegmentV1(version) + if err != nil || canonical != value { + return "", fmt.Errorf("encoded tool version is not canonical") + } + return version, nil +} + +func uppercaseHexValueV1(value byte) (byte, bool) { + switch { + case value >= '0' && value <= '9': + return value - '0', true + case value >= 'A' && value <= 'F': + return value - 'A' + 10, true + default: + return 0, false + } +} + +func validRecordTokenV1(value string) bool { + return value != "" && strings.TrimSpace(value) == value && !containsControlV1(value) +} + +func containsControlV1(value string) bool { + for _, character := range value { + if unicode.IsControl(character) { + return true + } + } + return false +} + +func validRecordSegmentV1(value string) bool { + if !validRecordTokenV1(value) || value == "." || value == ".." { + return false + } + for _, character := range value { + if character >= 'a' && character <= 'z' || character >= '0' && character <= '9' { + continue + } + switch character { + case '.', '+', '-': + default: + return false + } + } + return true +} + +func requireNonemptySortedStringsV1(field string, values []string) error { + if err := validateSortedUniqueStringsV1(field, values, false); err != nil { + return err + } + if len(values) == 0 { + return fmt.Errorf("%s must not be empty", field) + } + return nil +} diff --git a/internal/toolcatalog/records_decode_test.go b/internal/toolcatalog/records_decode_test.go new file mode 100644 index 00000000..6cdc0460 --- /dev/null +++ b/internal/toolcatalog/records_decode_test.go @@ -0,0 +1,421 @@ +package toolcatalog + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestDecodeRecordV1AcceptsEverySchema(t *testing.T) { + for _, value := range validRecordValuesV1() { + value := value + t.Run(fmt.Sprintf("%T", value), func(t *testing.T) { + payload, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + record, err := decodeRecordV1("record.json", payload) + if err != nil { + t.Fatal(err) + } + if record.ID != recordIDV1(value) || record.Schema == "" || record.Digest == "" || record.Value == nil { + t.Fatalf("decoded record = %#v", record) + } + }) + } +} + +func TestDecodeRecordV1UsesCanonicalSemanticIdentity(t *testing.T) { + first := []byte(`{ + "schema":"portable-tool-v1", + "id":"tool:demo", + "name":"demo", + "version_scheme":"semver", + "summary":"Demo tool", + "upstream":"https://example.com/demo", + "source":"https://example.com/source", + "license":"https://example.com/license", + "documentation":"https://example.com/docs", + "releases":[{"id":"tool:demo/releases/1.2.3/revisions/1/manifest","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}] +}`) + second := []byte(`{"releases":[{"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","id":"tool:demo/releases/1.2.3/revisions/1/manifest"}],"license":"https://example.com/license","documentation":"https://example.com/docs","source":"https://example.com/source","upstream":"https://example.com/demo","summary":"Demo tool","version_scheme":"semver","name":"demo","id":"tool:demo","schema":"portable-tool-v1"}`) + left, err := decodeRecordV1("first.json", first) + if err != nil { + t.Fatal(err) + } + right, err := decodeRecordV1("second.json", second) + if err != nil { + t.Fatal(err) + } + if left.Digest != right.Digest { + t.Fatalf("semantic digests differ: %s != %s", left.Digest, right.Digest) + } +} + +func TestDecodeRecordV1RejectsUnknownSchemaFieldAndMissingHeader(t *testing.T) { + for _, test := range []struct { + name string + payload string + want string + }{ + {name: "unknown schema", payload: `{"schema":"portable-tool-future-v1","id":"tool:demo"}`, want: "unsupported schema"}, + {name: "unknown field", payload: `{"schema":"portable-tool-v1","id":"tool:demo","name":"demo","version_scheme":"semver","summary":"x","upstream":"https://example.com","source":"https://example.com/source","license":"https://example.com/license","documentation":"https://example.com/docs","releases":[],"extra":true}`, want: "unknown field"}, + {name: "deterministic first unknown field", payload: `{"schema":"portable-tool-v1","id":"tool:demo","name":"demo","version_scheme":"semver","summary":"x","upstream":"https://example.com","source":"https://example.com/source","license":"https://example.com/license","documentation":"https://example.com/docs","releases":[],"z_extra":true,"a_extra":true}`, want: `unknown field "a_extra"`}, + {name: "case-variant field", payload: `{"schema":"portable-tool-v1","Schema":"portable-tool-v1","id":"tool:demo","name":"demo","version_scheme":"semver","summary":"x","upstream":"https://example.com","source":"https://example.com/source","license":"https://example.com/license","documentation":"https://example.com/docs","releases":[]}`, want: `unknown field "Schema"`}, + {name: "nested case-variant field", payload: `{"schema":"portable-tool-v1","id":"tool:demo","name":"demo","version_scheme":"semver","summary":"x","upstream":"https://example.com","source":"https://example.com/source","license":"https://example.com/license","documentation":"https://example.com/docs","releases":[{"ID":"tool:demo/releases/1/revisions/1/manifest","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`, want: `unknown field "ID"`}, + {name: "null binding", payload: `{"schema":"portable-tool-release-contract-v1","id":"tool:demo/releases/1/contract","contexts":["build"],"supported_reploy":">=0.0.0","binding":null,"selections":{"dimensions":[],"combinations":[]},"exports":[],"resolver_primitives":["https-sha256"],"compatibility_constraints":[]}`, want: "JSON null is not valid"}, + {name: "missing binding options", payload: `{"schema":"portable-tool-release-contract-v1","id":"tool:demo/releases/1/contract","contexts":["build"],"supported_reploy":">=0.0.0","binding":{},"selections":{"dimensions":[],"combinations":[]},"exports":[],"resolver_primitives":["https-sha256"],"compatibility_constraints":[]}`, want: `required field "options" is missing`}, + {name: "missing schema", payload: `{"id":"tool:demo"}`, want: "schema is required"}, + {name: "missing ID", payload: `{"schema":"portable-tool-v1"}`, want: "ID is required"}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := decodeRecordV1("record.json", []byte(test.payload)) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestValidateExactJSONMembersV1SortsMapKeys(t *testing.T) { + target := reflect.TypeOf(map[string]struct { + Value string `json:"value"` + }{}) + payload := json.RawMessage(`{"z":{"value":"ok","z_extra":true},"a":{"value":"ok","a_extra":true}}`) + err := validateExactJSONMembersV1(payload, target) + if err == nil || !strings.Contains(err.Error(), `unknown field "a_extra"`) { + t.Fatalf("error = %v, want deterministic first map-key error", err) + } +} + +func TestValidateStrictJSONV1RejectsAmbiguousAndOversizedInput(t *testing.T) { + tooManyValues := "[" + strings.Repeat("null,", maxDefinitionJSONMembers) + "null]" + tooDeep := strings.Repeat("[", maxDefinitionJSONDepth+2) + "null" + strings.Repeat("]", maxDefinitionJSONDepth+2) + tooDeepEmptyArray := strings.Repeat("[", maxDefinitionJSONDepth+1) + strings.Repeat("]", maxDefinitionJSONDepth+1) + tooDeepEmptyObject := strings.Repeat(`{"value":`, maxDefinitionJSONDepth) + `{}` + strings.Repeat("}", maxDefinitionJSONDepth) + tests := []struct { + name string + payload []byte + want string + }{ + {name: "empty", payload: nil, want: "record size"}, + {name: "invalid UTF-8", payload: []byte{'{', '"', 0xff, '"', ':', 'n', 'u', 'l', 'l', '}'}, want: "valid UTF-8"}, + {name: "duplicate member", payload: []byte(`{"a":null,"a":null}`), want: "duplicate object member"}, + {name: "number", payload: []byte(`{"size":1}`), want: "JSON numbers are not supported"}, + {name: "unpaired high surrogate", payload: []byte(`{"value":"\ud800"}`), want: "unpaired UTF-16 surrogate"}, + {name: "unpaired low surrogate", payload: []byte(`{"value":"\udc00"}`), want: "unpaired UTF-16 surrogate"}, + {name: "high surrogate without low", payload: []byte(`{"value":"\ud800\u0041"}`), want: "unpaired UTF-16 surrogate"}, + {name: "trailing value", payload: []byte(`{} {}`), want: "trailing JSON token"}, + {name: "too deep", payload: []byte(tooDeep), want: "JSON nesting exceeds"}, + {name: "too deep empty array", payload: []byte(tooDeepEmptyArray), want: "JSON nesting exceeds"}, + {name: "too deep empty object", payload: []byte(tooDeepEmptyObject), want: "JSON nesting exceeds"}, + {name: "too many members", payload: []byte(tooManyValues), want: "JSON member count exceeds"}, + {name: "long string", payload: []byte(`{"value":"` + strings.Repeat("x", maxDefinitionJSONStringBytes+1) + `"}`), want: "JSON string exceeds"}, + {name: "large file", payload: bytes.Repeat([]byte{' '}, maxDefinitionFileBytes+1), want: "record size"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateStrictJSONV1(test.payload) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want substring %q", err, test.want) + } + }) + } + for _, payload := range [][]byte{[]byte(`{"value":"\ud83d\ude00"}`), []byte(`{"value":"�"}`)} { + if err := validateStrictJSONV1(payload); err != nil { + t.Fatalf("valid Unicode payload %q: %v", payload, err) + } + } + atDepthLimit := strings.Repeat("[", maxDefinitionJSONDepth) + strings.Repeat("]", maxDefinitionJSONDepth) + if err := validateStrictJSONV1([]byte(atDepthLimit)); err != nil { + t.Fatalf("empty containers at depth limit: %v", err) + } +} + +func TestReleaseVersionSegmentsAreReversibleAndCanonical(t *testing.T) { + for version, want := range map[string]string{ + "1.2.3": "1.2.3", + "1!2": "1%212", + "A_B": "A_B", + ".": "%2E", + "..": "%2E%2E", + "雪": "%E9%9B%AA", + } { + encoded, err := encodeToolVersionSegmentV1(version) + if err != nil || encoded != want { + t.Fatalf("encode %q = %q, %v; want %q", version, encoded, err, want) + } + decoded, err := decodeToolVersionSegmentV1(encoded) + if err != nil || decoded != version { + t.Fatalf("decode %q = %q, %v; want %q", encoded, decoded, err, version) + } + } + for _, encoded := range []string{"", "1%", "1%2f2", "1%312", ".", "..", "%FF"} { + if _, err := decodeToolVersionSegmentV1(encoded); err == nil { + t.Fatalf("noncanonical encoded version %q was accepted", encoded) + } + } +} + +func TestValidateSourceURLV1RequiresCanonicalCredentialFreeHTTPS(t *testing.T) { + for _, valid := range []string{ + "https://example.com/", + "https://example.com/releases/jdk-21.0.12%2B8/archive.tar.gz", + "https://example.com/releases/archive%23checksum", + "https://example.com:8443/archive.tar.gz", + "https://0xrelease.example.com/archive.tar.gz", + } { + if err := validateSourceURLV1(valid); err != nil { + t.Fatalf("valid URL %q: %v", valid, err) + } + } + for _, invalid := range []string{ + "http://example.com/archive", + "https://user:password@example.com/archive", + "https://example.com/archive?", + "https://example.com/archive?token=secret", + "https://example.com/archive#", + "https://example.com/archive#checksum", + "https://EXAMPLE.com/archive", + "https://example.com:/archive", + "https://example.com.:443/archive", + "https://example.com:443/archive", + "https://example.com:0443/archive", + "https://127.000.000.001/archive", + "https://127.1/archive", + "https://017700000001/archive", + "https://0x7f000001/archive", + "https://0x7f.0x0.0x0.0x1/archive", + "https://[fe80::1%25eth0]/archive", + "https://example.com/releases/jdk%2b21/archive", + "https://example.com/releases/../archive", + "https://éxample.com/archive", + "https://example.com", + "https://example.com/%61", + "https://[2001:0db8:0:0:0:0:0:1]/archive", + } { + if err := validateSourceURLV1(invalid); err == nil { + t.Fatalf("invalid URL %q was accepted", invalid) + } + } + canonicalPlain, err := canonicalSourceURLV1("https://example.com/a") + if err != nil { + t.Fatal(err) + } + canonicalEscaped, err := canonicalSourceURLV1("https://example.com/%61") + if err != nil { + t.Fatal(err) + } + if canonicalPlain != canonicalEscaped { + t.Fatalf("equivalent URL identities differ: %q != %q", canonicalPlain, canonicalEscaped) + } + canonicalRoot, err := canonicalSourceURLV1("https://example.com/") + if err != nil { + t.Fatal(err) + } + canonicalEmptyPath, err := canonicalSourceURLV1("https://example.com") + if err != nil { + t.Fatal(err) + } + if canonicalRoot != canonicalEmptyPath { + t.Fatalf("root URL identities differ: %q != %q", canonicalRoot, canonicalEmptyPath) + } + canonicalIPv6, err := canonicalSourceURLV1("https://[2001:db8::1]/archive") + if err != nil { + t.Fatal(err) + } + canonicalExpandedIPv6, err := canonicalSourceURLV1("https://[2001:0db8:0:0:0:0:0:1]/archive") + if err != nil { + t.Fatal(err) + } + if canonicalIPv6 != canonicalExpandedIPv6 { + t.Fatalf("IPv6 URL identities differ: %q != %q", canonicalIPv6, canonicalExpandedIPv6) + } + canonicalReserved, err := canonicalSourceURLV1("https://example.com/%2B") + if err != nil { + t.Fatal(err) + } + if canonicalReserved == "https://example.com/+" { + t.Fatal("reserved percent escape was normalized") + } +} + +func TestDecodeValidationEvidenceV1(t *testing.T) { + evidence := *validValidationEvidenceV1() + payload, err := json.Marshal(evidence) + if err != nil { + t.Fatal(err) + } + if _, err := decodeValidationEvidenceV1("evidence.json", payload); err != nil { + t.Fatal(err) + } + caseVariantPayload := bytes.Replace(payload, []byte(`"schema"`), []byte(`"Schema"`), 1) + if _, err := decodeValidationEvidenceV1("evidence.json", caseVariantPayload); err == nil || !strings.Contains(err.Error(), `unknown field "Schema"`) { + t.Fatalf("case-variant evidence error = %v", err) + } + // Rejecting a semantically invalid result is evidence validation, which this + // slice deliberately excludes; its coverage arrives with that validation. +} + +func releaseScopedID(name string) string { + return "tool:demo/releases/1.2.3/payloads/" + name +} + +func TestRecordPathsRejectBackslashes(t *testing.T) { + // path.Clean and path.IsAbs treat a backslash as an ordinary character, so a + // Windows-style separator would otherwise pass here and fail only later in + // providerstore.ArtifactDescriptor.Validate, which forbids it explicitly. + for _, value := range []string{`tools\\demo.zip`, `tools\\demo\\chromium.zip`, `a\\b`} { + if err := validateRecordPathV1(value, false); err == nil { + t.Errorf("validateRecordPathV1(%q) accepted a backslash", value) + } + } + for _, value := range []string{`/opt\\demo/bin/demo`, `/opt/demo\\bin`} { + if err := validateAbsoluteRecordPathV1(value); err == nil { + t.Errorf("validateAbsoluteRecordPathV1(%q) accepted a backslash", value) + } + } + if err := validateRecordPathV1("tools/demo.zip", false); err != nil { + t.Errorf("canonical relative path rejected: %v", err) + } + if err := validateRecordPathV1(".", true); err != nil { + t.Errorf("allowed current-directory path rejected: %v", err) + } + if err := validateRecordPathV1(".", false); err == nil { + t.Error("current-directory path accepted when disallowed") + } + if err := validateAbsoluteRecordPathV1("/opt/demo/bin/demo"); err != nil { + t.Errorf("canonical absolute path rejected: %v", err) + } +} + +func TestCanonicalCollectionsAreBoundedSortedAndUnique(t *testing.T) { + sorted := []RecordReferenceV1{recordTestReference(releaseScopedID("a")), recordTestReference(releaseScopedID("b"))} + if err := validateReferenceListV1("references", sorted); err != nil { + t.Fatalf("sorted unique references: %v", err) + } + oversized := make([]RecordReferenceV1, maxDefinitionReferences+1) + for index := range oversized { + oversized[index] = recordTestReference(releaseScopedID(fmt.Sprintf("p%06d", index))) + } + for _, testCase := range []struct { + name string + references []RecordReferenceV1 + want string + }{ + {name: "nil", references: nil, want: "must use an array"}, + {name: "over limit", references: oversized, want: "at most"}, + {name: "unsorted", references: []RecordReferenceV1{recordTestReference(releaseScopedID("b")), recordTestReference(releaseScopedID("a"))}, want: "references"}, + {name: "duplicate", references: []RecordReferenceV1{recordTestReference(releaseScopedID("a")), recordTestReference(releaseScopedID("a"))}, want: "references"}, + } { + err := validateReferenceListV1("references", testCase.references) + if err == nil || !strings.Contains(err.Error(), testCase.want) { + t.Errorf("%s references error = %v", testCase.name, err) + } + } + + if err := validateSortedUniqueStringsV1("values", []string{"alpha", "beta"}, false); err != nil { + t.Fatalf("sorted unique strings: %v", err) + } + for _, testCase := range []struct { + name string + values []string + }{ + {name: "nil", values: nil}, + {name: "empty value", values: []string{""}}, + {name: "control character", values: []string{"alpha\nbeta"}}, + {name: "untrimmed", values: []string{" alpha"}}, + {name: "unsorted", values: []string{"beta", "alpha"}}, + {name: "duplicate", values: []string{"alpha", "alpha"}}, + } { + if err := validateSortedUniqueStringsV1("values", testCase.values, false); err == nil { + t.Errorf("%s strings error = nil", testCase.name) + } + } + if err := requireNonemptySortedStringsV1("values", []string{"alpha"}); err != nil { + t.Fatalf("nonempty sorted strings: %v", err) + } + if err := requireNonemptySortedStringsV1("values", []string{}); err == nil { + t.Fatal("empty string collection was accepted as nonempty") + } +} + +func TestCanonicalRecordIdentifiersReferencesAndDecimals(t *testing.T) { + for _, value := range []string{ + "tool:demo", + "tool:demo/releases/1.2.3/payloads/demo", + "tool:demo/releases/1%212/payloads/demo", + } { + if err := validateRecordIDV1(value); err != nil { + t.Errorf("canonical record ID %q: %v", value, err) + } + } + for _, value := range []string{ + "", + "demo", + "tool:Demo", + "tool:demo/releases/1.2.3", + "tool:demo/releases/1%2f2/payloads/demo", + "tool:demo/releases/%31/payloads/demo", + "tool:demo/releases/1.2.3/payloads/%41", + "tool:demo/releases/1.2.3/../demo", + } { + if err := validateRecordIDV1(value); err == nil { + t.Errorf("noncanonical record ID %q was accepted", value) + } + } + + validReference := recordTestReference("tool:demo/releases/1.2.3/payloads/demo") + if err := validateRecordReferenceV1(validReference); err != nil { + t.Fatalf("canonical record reference: %v", err) + } + invalidDigest := validReference + invalidDigest.Digest = "" + if err := validateRecordReferenceV1(invalidDigest); err == nil { + t.Fatal("record reference with an invalid digest was accepted") + } + + for _, testCase := range []struct { + value string + positive bool + }{ + {value: "0"}, + {value: "1", positive: true}, + {value: "9223372036854775807", positive: true}, + } { + if err := validateCanonicalDecimalV1("value", testCase.value, testCase.positive); err != nil { + t.Errorf("canonical decimal %q: %v", testCase.value, err) + } + } + for _, testCase := range []struct { + value string + positive bool + }{ + {value: ""}, + {value: "00"}, + {value: "01"}, + {value: "-1"}, + {value: "0", positive: true}, + {value: "9223372036854775808", positive: true}, + } { + if err := validateCanonicalDecimalV1("value", testCase.value, testCase.positive); err == nil { + t.Errorf("noncanonical decimal %q was accepted", testCase.value) + } + } +} + +func TestCanonicalRecordSegments(t *testing.T) { + for _, value := range []string{"demo", "demo-1.2+3"} { + if !validRecordSegmentV1(value) { + t.Errorf("canonical record segment %q was rejected", value) + } + } + for _, value := range []string{"", ".", "..", "Demo", "demo_name", "demo/name", "demo name"} { + if validRecordSegmentV1(value) { + t.Errorf("noncanonical record segment %q was accepted", value) + } + } +}