diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index 602f45ff22b..f85b8dad651 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -2193,7 +2193,7 @@ func (provider *BedrockProvider) ResponsesStream(ctx *schemas.BifrostContext, po } // Embedding generates embeddings for the given input text(s) using Amazon Bedrock. -// Supports Titan and Cohere embedding models. Returns a BifrostResponse containing the embedding(s) and any error that occurred. +// Supports Titan, Cohere and Nova embedding models. Returns a BifrostResponse containing the embedding(s) and any error that occurred. func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { if err := providerUtils.CheckOperationAllowed(schemas.Bedrock, provider.customProviderConfig, schemas.EmbeddingRequest); err != nil { return nil, err @@ -2240,6 +2240,19 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model) + case "nova": + jsonData, bifrostError = providerUtils.CheckContextAndGetRequestBody( + ctx, + request, + func() (providerUtils.RequestBodyWithExtraParams, error) { + return ToBedrockNovaEmbeddingRequest(request) + }) + if bifrostError != nil { + return nil, bifrostError + } + path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) + rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model) + default: return nil, providerUtils.NewConfigurationError("unsupported embedding model type") } @@ -2302,6 +2315,35 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche } bifrostResponse = converted bifrostResponse.Model = request.Model + + case "nova": + var novaResp BedrockNovaEmbeddingResponse + novaParseTracer, novaParseHandle := providerUtils.StartResponseParseSpan(ctx) + umErr := sonic.Unmarshal(rawResponse, &novaResp) + if novaParseTracer != nil { + if umErr != nil { + novaParseTracer.EndSpan(novaParseHandle, schemas.SpanStatusError, umErr.Error()) + } else { + novaParseTracer.EndSpan(novaParseHandle, schemas.SpanStatusOk, "") + } + } + if umErr != nil { + return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError("error parsing Nova embedding response", umErr), jsonData, rawResponse, provider.sendBackRawRequest, provider.sendBackRawResponse, latency) + } + novaConvTracer, novaConvHandle := providerUtils.StartResponseConvertorSpan(ctx) + converted, convErr := novaResp.ToBifrostEmbeddingResponse() + if novaConvTracer != nil { + if convErr != nil { + novaConvTracer.EndSpan(novaConvHandle, schemas.SpanStatusError, convErr.Error()) + } else { + novaConvTracer.EndSpan(novaConvHandle, schemas.SpanStatusOk, "") + } + } + if convErr != nil { + return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError("error parsing Nova embedding response", convErr), jsonData, rawResponse, provider.sendBackRawRequest, provider.sendBackRawResponse, latency) + } + bifrostResponse = converted + bifrostResponse.Model = request.Model } // Titan admits only float/binary and Cohere v3 has no base64 at all, so a base64 request diff --git a/core/providers/bedrock/embedding.go b/core/providers/bedrock/embedding.go index db8f5fff66c..38282d680f6 100644 --- a/core/providers/bedrock/embedding.go +++ b/core/providers/bedrock/embedding.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "math" + "mime" + "path/filepath" "strconv" "strings" @@ -371,6 +373,291 @@ func ToBedrockCohereEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest return req, nil } +// novaEmbeddingFormats maps the media types and file extensions a caller is likely to +// carry onto Nova's per-modality format enum. AWS checks the declared format against +// the bytes it detects, so an underivable format is refused here rather than guessed. +var novaEmbeddingFormats = map[schemas.EmbeddingContentPartType]map[string]string{ + schemas.EmbeddingContentPartTypeImage: { + "image/png": "png", "png": "png", + "image/jpeg": "jpeg", "image/jpg": "jpeg", "jpeg": "jpeg", "jpg": "jpeg", + "image/webp": "webp", "webp": "webp", + "image/gif": "gif", "gif": "gif", + }, + schemas.EmbeddingContentPartTypeAudio: { + "audio/mpeg": "mp3", "audio/mp3": "mp3", "mp3": "mp3", + "audio/wav": "wav", "audio/x-wav": "wav", "audio/wave": "wav", "wav": "wav", + "audio/ogg": "ogg", "ogg": "ogg", + }, + schemas.EmbeddingContentPartTypeVideo: { + "video/mp4": "mp4", "mp4": "mp4", + "video/quicktime": "mov", "mov": "mov", + "video/x-matroska": "mkv", "mkv": "mkv", + "video/webm": "webm", "webm": "webm", + "video/x-flv": "flv", "flv": "flv", + "video/mpeg": "mpeg", "mpeg": "mpeg", "mpg": "mpeg", + "video/x-ms-wmv": "wmv", "wmv": "wmv", + "video/3gpp": "three_gp", "3gp": "three_gp", + }, +} + +// novaEmbeddingMediaTypes is the reverse of novaEmbeddingFormats, used by the Bedrock +// integration to hand a native format name back as a media type. Every value here is a +// key of the table above, so a format survives the round trip unchanged. +// +// mpg and 3gp are inbound spellings only. AWS's reference documents 3gp, but the API +// rejects it and takes three_gp, so a body copied from the reference normalizes to the +// spelling that works rather than failing format derivation on the way out. +var novaEmbeddingMediaTypes = map[string]string{ + "png": "image/png", "jpeg": "image/jpeg", "webp": "image/webp", "gif": "image/gif", + "mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg", + "mp4": "video/mp4", "mov": "video/quicktime", "mkv": "video/x-matroska", + "webm": "video/webm", "flv": "video/x-flv", "mpeg": "video/mpeg", "mpg": "video/mpeg", + "wmv": "video/x-ms-wmv", "three_gp": "video/3gpp", "3gp": "video/3gpp", +} + +// novaEmbeddingFormatHints names the media types accepted per modality, for the error +// raised when neither mime_type nor the URL says which one a part carries. +var novaEmbeddingFormatHints = map[schemas.EmbeddingContentPartType]string{ + schemas.EmbeddingContentPartTypeImage: "image/png, image/jpeg, image/webp or image/gif", + schemas.EmbeddingContentPartTypeAudio: "audio/mpeg, audio/wav or audio/ogg", + schemas.EmbeddingContentPartTypeVideo: "video/mp4, video/quicktime, video/x-matroska, video/webm, video/x-flv, video/mpeg, video/x-ms-wmv or video/3gpp", +} + +// novaEmbeddingFormat resolves Nova's format name from the declared media type, falling +// back to the extension of an s3:// key when no type was given. +func novaEmbeddingFormat(partType schemas.EmbeddingContentPartType, mimeType, rawURL string) (string, error) { + table := novaEmbeddingFormats[partType] + if table == nil { + return "", providerUtils.InvalidRequestErrorf("amazon Nova embedding models do not support %q parts", partType) + } + if mimeType != "" { + if parsed, _, err := mime.ParseMediaType(mimeType); err == nil { + mimeType = parsed + } + if format, ok := table[strings.ToLower(strings.TrimSpace(mimeType))]; ok { + return format, nil + } + } + if rawURL != "" { + path := rawURL + if i := strings.IndexAny(path, "?#"); i >= 0 { + path = path[:i] + } + if format, ok := table[strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))]; ok { + return format, nil + } + } + return "", providerUtils.InvalidRequestErrorf("cannot determine the %s format for amazon Nova: pass mime_type as one of %s", partType, novaEmbeddingFormatHints[partType]) +} + +// novaEmbeddingMediaPart converts one media part into Nova's format and source pair. +// s3:// travels as s3Location, which Nova resolves itself; inline data travels as bytes. +// An http(s) URL has no home here - Nova fetches neither - so it is refused with the +// two shapes that do work. +func novaEmbeddingMediaPart(partType schemas.EmbeddingContentPartType, media *schemas.EmbeddingMediaPart) (string, BedrockNovaEmbeddingSource, error) { + var empty BedrockNovaEmbeddingSource + if err := media.Validate(); err != nil { + return "", empty, providerUtils.InvalidRequestErrorf("%s", err) + } + + mimeType := "" + if media.MIMEType != nil { + mimeType = *media.MIMEType + } + + if media.Data != nil { + data := *media.Data + if strings.HasPrefix(data, "data:") { + mediaType, _, payload, ok := schemas.ParseDataURL(data) + if !ok { + return "", empty, providerUtils.InvalidRequestErrorf("%s part carries a malformed data url", partType) + } + data = payload + if mimeType == "" { + mimeType = mediaType + } + } + if data == "" { + return "", empty, providerUtils.InvalidRequestErrorf("%s part has empty data", partType) + } + format, err := novaEmbeddingFormat(partType, mimeType, "") + if err != nil { + return "", empty, err + } + return format, BedrockNovaEmbeddingSource{Bytes: &data}, nil + } + + url := *media.URL + s3Location, ok := bedrockS3LocationFromURL(url) + if !ok { + if strings.HasPrefix(url, "s3://") { + return "", empty, providerUtils.InvalidRequestErrorf("invalid s3:// %s reference %q: expected s3://bucket/key", partType, url) + } + return "", empty, providerUtils.InvalidRequestErrorf("amazon Nova embedding models read media from s3:// or inline base64 only, so %q cannot be used; send the %s inline or presign it into S3", url, partType) + } + format, err := novaEmbeddingFormat(partType, mimeType, url) + if err != nil { + return "", empty, err + } + return format, BedrockNovaEmbeddingSource{S3Location: s3Location}, nil +} + +// ToBedrockNovaEmbeddingRequest converts a Bifrost embedding request to Bedrock Nova format. +// SINGLE_EMBEDDING embeds one content item carrying one modality, so a batch or a mixed +// item is refused here rather than turned into an AWS schema error naming a key the +// caller never wrote. +func ToBedrockNovaEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest) (*BedrockNovaEmbeddingRequest, error) { + if bifrostReq == nil { + return nil, fmt.Errorf("bifrost embedding request is nil") + } + + if len(bifrostReq.Input) == 0 { + return nil, providerUtils.InvalidRequestErrorf("no input provided for Nova embedding") + } + + if len(bifrostReq.Input) != 1 { + return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models support exactly one content item per request; got %d", len(bifrostReq.Input)) + } + + if err := schemas.EmbeddingInput(bifrostReq.Input).RejectPerItemParams("amazon Nova"); err != nil { + return nil, providerUtils.InvalidRequestErrorf("%s", err) + } + + params := &BedrockNovaSingleEmbeddingParams{EmbeddingPurpose: BedrockNovaEmbeddingPurposeGenericIndex} + truncationMode := BedrockNovaTruncationModeEnd + videoMode := BedrockNovaVideoModeAudioVideoCombined + var detailLevel *BedrockNovaDetailLevel + extra := map[string]interface{}{} + + if bifrostReq.Params != nil { + params.EmbeddingDimension = bifrostReq.Params.Dimensions + if bifrostReq.Params.TaskType != nil && *bifrostReq.Params.TaskType != "" { + params.EmbeddingPurpose = BedrockNovaEmbeddingPurpose(*bifrostReq.Params.TaskType) + } + for k, v := range bifrostReq.Params.ExtraParams { + s, isString := v.(string) + switch { + case k == BedrockNovaExtraParamEmbeddingPurpose && isString && s != "": + params.EmbeddingPurpose = BedrockNovaEmbeddingPurpose(s) + case k == BedrockNovaExtraParamTruncationMode && isString && s != "": + truncationMode = BedrockNovaTruncationMode(s) + case k == BedrockNovaExtraParamEmbeddingMode && isString && s != "": + videoMode = BedrockNovaVideoMode(s) + case k == BedrockNovaExtraParamDetailLevel && isString && s != "": + detailLevel = schemas.Ptr(BedrockNovaDetailLevel(s)) + default: + extra[k] = v + } + } + } + + var sb strings.Builder + hasText := false + modalities := 0 + for _, part := range bifrostReq.Input[0].Content { + switch part.Type { + case schemas.EmbeddingContentPartTypeText: + if part.Text == nil { + return nil, providerUtils.InvalidRequestErrorf("text part carries no text") + } + if hasText { + sb.WriteString("\n") + } else { + hasText = true + modalities++ + } + sb.WriteString(*part.Text) + case schemas.EmbeddingContentPartTypeImage, schemas.EmbeddingContentPartTypeAudio, schemas.EmbeddingContentPartTypeVideo: + media := part.Image + if part.Type == schemas.EmbeddingContentPartTypeAudio { + media = part.Audio + } else if part.Type == schemas.EmbeddingContentPartTypeVideo { + media = part.Video + } + format, source, err := novaEmbeddingMediaPart(part.Type, media) + if err != nil { + return nil, err + } + switch part.Type { + case schemas.EmbeddingContentPartTypeImage: + params.Image = &BedrockNovaEmbeddingImage{Format: format, Source: source, DetailLevel: detailLevel} + case schemas.EmbeddingContentPartTypeAudio: + params.Audio = &BedrockNovaEmbeddingMedia{Format: format, Source: source} + default: + params.Video = &BedrockNovaEmbeddingVideo{Format: format, EmbeddingMode: videoMode, Source: source} + } + modalities++ + default: + return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models do not support %q parts", part.Type) + } + } + + if modalities == 0 { + return nil, providerUtils.InvalidRequestErrorf("no input provided for Nova embedding") + } + if modalities > 1 { + return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models embed one modality per request: text, image, audio or video, not a combination") + } + + if hasText { + value := sb.String() + if strings.TrimSpace(value) == "" { + return nil, providerUtils.InvalidRequestErrorf("text part carries no text") + } + params.Text = &BedrockNovaEmbeddingText{TruncationMode: truncationMode, Value: &value} + } + + req := &BedrockNovaEmbeddingRequest{ + TaskType: TaskTypeSingleEmbedding, + SingleEmbeddingParams: params, + } + if len(extra) > 0 { + req.ExtraParams = extra + } + return req, nil +} + +// novaEmbeddingModalities maps Nova's per-vector embeddingType label onto the canonical +// modality. AUDIO_VIDEO_COMBINED is one vector covering both, which the separate mode +// splits into an AUDIO and a VIDEO entry. +var novaEmbeddingModalities = map[string]schemas.EmbeddingModality{ + "TEXT": schemas.EmbeddingModalityText, + "IMAGE": schemas.EmbeddingModalityImage, + "AUDIO": schemas.EmbeddingModalityAudio, + "VIDEO": schemas.EmbeddingModalityVideo, + "AUDIO_VIDEO_COMBINED": schemas.EmbeddingModalityAudioVideo, +} + +// novaEmbeddingTypes is the reverse of novaEmbeddingModalities, for rebuilding the +// native envelope on the Bedrock integration's invoke route. +var novaEmbeddingTypes = map[schemas.EmbeddingModality]string{ + schemas.EmbeddingModalityText: "TEXT", + schemas.EmbeddingModalityImage: "IMAGE", + schemas.EmbeddingModalityAudio: "AUDIO", + schemas.EmbeddingModalityVideo: "VIDEO", + schemas.EmbeddingModalityAudioVideo: "AUDIO_VIDEO_COMBINED", +} + +// ToBifrostEmbeddingResponse converts a Bedrock Nova embedding response to Bifrost format. +// Usage is left unset: Nova reports input tokens only in the +// X-Amzn-Bedrock-Input-Token-Count header, which the caller backfills from. +func (r *BedrockNovaEmbeddingResponse) ToBifrostEmbeddingResponse() (*schemas.BifrostEmbeddingResponse, error) { + if r == nil { + return nil, fmt.Errorf("nil Bedrock Nova embedding response") + } + + bifrostResponse := &schemas.BifrostEmbeddingResponse{Object: "list"} + for i, embedding := range r.Embeddings { + bifrostResponse.Data = append(bifrostResponse.Data, schemas.EmbeddingData{ + Index: i, + Object: "embedding", + Modality: novaEmbeddingModalities[embedding.EmbeddingType], + Embedding: schemas.EmbeddingStruct{EmbeddingArray: embedding.Embedding}, + }) + } + return bifrostResponse, nil +} + // DetermineEmbeddingModelType determines the embedding model type for the // current attempt. It consults the resolved alias family first // (model_family / model_name / model_id / alias key) and falls back to the @@ -378,6 +665,8 @@ func ToBedrockCohereEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest // Bedrock deployment that's tagged with the right family routes correctly. func DetermineEmbeddingModelType(ctx *schemas.BifrostContext, model string) (string, error) { switch { + case schemas.IsNovaModelFamily(ctx, model): + return "nova", nil case schemas.IsTitanModelFamily(ctx, model): return "titan", nil case schemas.IsCohereModelFamily(ctx, model): @@ -481,4 +770,4 @@ func (r *BedrockCohereEmbeddingResponse) ToBifrostEmbeddingResponse() (*schemas. } return bifrostResponse, nil -} +} \ No newline at end of file diff --git a/core/providers/bedrock/embedding_test.go b/core/providers/bedrock/embedding_test.go index eec81d987aa..61afc0f9117 100644 --- a/core/providers/bedrock/embedding_test.go +++ b/core/providers/bedrock/embedding_test.go @@ -855,3 +855,568 @@ func TestBedrockInvokeEmbeddingRejectsInvalidInputBlocks(t *testing.T) { require.Len(t, req.Input, 1) assert.Len(t, req.Input[0].Content, 2) } + +// Nova is a third envelope alongside Titan's and Cohere's: a taskType plus a params +// object holding exactly one modality, every knob of which AWS requires and none of +// which it defaults. Probed live against amazon.nova-2-multimodal-embeddings-v1:0. +func TestToBedrockNovaEmbeddingRequest(t *testing.T) { + const novaModel = "amazon.nova-2-multimodal-embeddings-v1:0" + redPixelPNG := "iVBORw0KGgoAAAANSUhEUg==" + + textRequest := func(params *schemas.EmbeddingParameters) *schemas.BifrostEmbeddingRequest { + text := "alpha kestrel harbor" + return &schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeText, Text: &text}, + }}}, + Params: params, + } + } + + t.Run("text produces the documented wire body", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(textRequest(nil)) + require.NoError(t, err) + wireBody, err := json.Marshal(req) + require.NoError(t, err) + assert.JSONEq(t, `{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "GENERIC_INDEX", + "text": {"truncationMode": "END", "value": "alpha kestrel harbor"} + } + }`, string(wireBody)) + }) + + t.Run("dimensions map onto embeddingDimension", func(t *testing.T) { + dimensions := 1024 + req, err := ToBedrockNovaEmbeddingRequest(textRequest(&schemas.EmbeddingParameters{Dimensions: &dimensions})) + require.NoError(t, err) + require.NotNil(t, req.SingleEmbeddingParams.EmbeddingDimension) + assert.Equal(t, 1024, *req.SingleEmbeddingParams.EmbeddingDimension) + }) + + // embeddingPurpose is Nova's own name for task_type and arrives only through the + // Bedrock integration, so it wins - the same precedence embeddingTypes has on Titan. + t.Run("task_type becomes embeddingPurpose and embeddingPurpose outranks it", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(textRequest(&schemas.EmbeddingParameters{ + TaskType: schemas.Ptr("CLASSIFICATION"), + })) + require.NoError(t, err) + // Compared against the wire spelling, not the const, so a const whose value drifts + // from AWS's enum still fails here. + assert.Equal(t, BedrockNovaEmbeddingPurpose("CLASSIFICATION"), req.SingleEmbeddingParams.EmbeddingPurpose) + + req, err = ToBedrockNovaEmbeddingRequest(textRequest(&schemas.EmbeddingParameters{ + TaskType: schemas.Ptr("CLASSIFICATION"), + ExtraParams: map[string]interface{}{"embeddingPurpose": "DOCUMENT_RETRIEVAL"}, + })) + require.NoError(t, err) + assert.Equal(t, BedrockNovaEmbeddingPurpose("DOCUMENT_RETRIEVAL"), req.SingleEmbeddingParams.EmbeddingPurpose) + }) + + t.Run("truncationMode is overridable and stays out of ExtraParams", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(textRequest(&schemas.EmbeddingParameters{ + ExtraParams: map[string]interface{}{"truncationMode": "START", "someOtherKey": 1}, + })) + require.NoError(t, err) + assert.Equal(t, BedrockNovaTruncationMode("START"), req.SingleEmbeddingParams.Text.TruncationMode) + assert.NotContains(t, req.ExtraParams, "truncationMode") + assert.Contains(t, req.ExtraParams, "someOtherKey") + }) + + t.Run("several text parts join into the one value Nova accepts", func(t *testing.T) { + first, second := "alpha", "kestrel" + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeText, Text: &first}, + {Type: schemas.EmbeddingContentPartTypeText, Text: &second}, + }}}, + }) + require.NoError(t, err) + require.NotNil(t, req.SingleEmbeddingParams.Text.Value) + assert.Equal(t, "alpha\nkestrel", *req.SingleEmbeddingParams.Text.Value) + }) + + t.Run("inline media travels as bytes with the format derived", func(t *testing.T) { + tests := []struct { + name string + part schemas.EmbeddingContentPart + format string + pickFrom func(*BedrockNovaSingleEmbeddingParams) (string, BedrockNovaEmbeddingSource) + }{ + { + name: "image data uri", + part: schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeImage, + Image: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:image/png;base64," + redPixelPNG)}}, + format: "png", + pickFrom: func(p *BedrockNovaSingleEmbeddingParams) (string, BedrockNovaEmbeddingSource) { + return p.Image.Format, p.Image.Source + }, + }, + { + name: "image bare base64 with mime_type", + part: schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeImage, + Image: &schemas.EmbeddingMediaPart{Data: &redPixelPNG, MIMEType: schemas.Ptr("image/jpeg")}}, + format: "jpeg", + pickFrom: func(p *BedrockNovaSingleEmbeddingParams) (string, BedrockNovaEmbeddingSource) { + return p.Image.Format, p.Image.Source + }, + }, + { + name: "audio", + part: schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeAudio, + Audio: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:audio/mpeg;base64,AA==")}}, + format: "mp3", + pickFrom: func(p *BedrockNovaSingleEmbeddingParams) (string, BedrockNovaEmbeddingSource) { + return p.Audio.Format, p.Audio.Source + }, + }, + { + name: "video", + part: schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeVideo, + Video: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:video/quicktime;base64,AA==")}}, + format: "mov", + pickFrom: func(p *BedrockNovaSingleEmbeddingParams) (string, BedrockNovaEmbeddingSource) { + return p.Video.Format, p.Video.Source + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{test.part}}}, + }) + require.NoError(t, err) + format, source := test.pickFrom(req.SingleEmbeddingParams) + assert.Equal(t, test.format, format) + require.NotNil(t, source.Bytes, "inline media travels as bytes") + assert.Nil(t, source.S3Location) + assert.NotContains(t, *source.Bytes, "data:", "the data URI prefix is stripped") + }) + } + }) + + // Nova resolves s3:// itself, the only Bedrock embedding model that does. + t.Run("s3 urls become an s3Location", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeImage, + Image: &schemas.EmbeddingMediaPart{URL: schemas.Ptr("s3://my-bucket/photos/red-square.png")}}, + }}}, + }) + require.NoError(t, err) + require.NotNil(t, req.SingleEmbeddingParams.Image.Source.S3Location) + assert.Equal(t, "s3://my-bucket/photos/red-square.png", req.SingleEmbeddingParams.Image.Source.S3Location.URI) + assert.Equal(t, "png", req.SingleEmbeddingParams.Image.Format, "the key's extension names the format when no mime_type does") + }) + + // embeddingMode is required on video and has no AWS default. AUDIO_VIDEO_SEPARATE + // asks for an audio and a video vector instead of one covering both. + t.Run("video carries an embeddingMode, overridable through extra params", func(t *testing.T) { + videoRequest := func(params *schemas.EmbeddingParameters) *schemas.BifrostEmbeddingRequest { + return &schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeVideo, + Video: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:video/mp4;base64,AA==")}}, + }}}, + Params: params, + } + } + req, err := ToBedrockNovaEmbeddingRequest(videoRequest(nil)) + require.NoError(t, err) + assert.Equal(t, BedrockNovaVideoMode("AUDIO_VIDEO_COMBINED"), req.SingleEmbeddingParams.Video.EmbeddingMode) + + req, err = ToBedrockNovaEmbeddingRequest(videoRequest(&schemas.EmbeddingParameters{ + ExtraParams: map[string]interface{}{"embeddingMode": "AUDIO_VIDEO_SEPARATE"}, + })) + require.NoError(t, err) + assert.Equal(t, BedrockNovaVideoMode("AUDIO_VIDEO_SEPARATE"), req.SingleEmbeddingParams.Video.EmbeddingMode) + }) + + t.Run("requests Nova cannot represent are refused here", func(t *testing.T) { + tests := []struct { + name string + input []schemas.EmbeddingInputItem + message string + }{ + { + name: "text and image in one item", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeText, Text: schemas.Ptr("a red square")}, + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:image/png;base64," + redPixelPNG)}}, + }}}, + message: "one modality per request", + }, + { + name: "two images in one item", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:image/png;base64," + redPixelPNG)}}, + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:image/png;base64," + redPixelPNG)}}, + }}}, + message: "one modality per request", + }, + { + name: "a batch of two items", + input: []schemas.EmbeddingInputItem{ + {Content: schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeText, Text: schemas.Ptr("alpha")}}}, + {Content: schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeText, Text: schemas.Ptr("kestrel")}}}, + }, + message: "exactly one content item per request", + }, + { + name: "a file part, which Nova has no modality for", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeFile, File: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:application/pdf;base64,AA==")}}, + }}}, + message: `do not support "file" parts`, + }, + { + name: "an https url Nova will not fetch", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{URL: schemas.Ptr("https://example.com/red.png")}}, + }}}, + message: "s3:// or inline base64 only", + }, + { + name: "an s3 uri naming no object", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{URL: schemas.Ptr("s3://my-bucket")}}, + }}}, + message: "expected s3://bucket/key", + }, + { + name: "media whose format nothing names", + input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeImage, Image: &schemas.EmbeddingMediaPart{Data: &redPixelPNG}}, + }}}, + message: "cannot determine the image format", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{Model: novaModel, Input: test.input}) + require.Error(t, err) + assert.Contains(t, err.Error(), test.message) + assert.True(t, providerUtils.IsInvalidRequestError(err), "caller faults must be 400s, got %v", err) + }) + } + }) +} + +func TestBedrockNovaEmbeddingResponse(t *testing.T) { + t.Run("each vector carries the modality that produced it", func(t *testing.T) { + var response BedrockNovaEmbeddingResponse + require.NoError(t, json.Unmarshal([]byte(`{"embeddings":[ + {"embedding":[0.1,0.2],"embeddingType":"AUDIO"}, + {"embedding":[0.3,0.4],"embeddingType":"VIDEO"} + ]}`), &response)) + + converted, err := response.ToBifrostEmbeddingResponse() + require.NoError(t, err) + require.Len(t, converted.Data, 2) + assert.Equal(t, schemas.EmbeddingModalityAudio, converted.Data[0].Modality) + assert.Equal(t, 0, converted.Data[0].Index) + assert.Equal(t, schemas.EmbeddingModalityVideo, converted.Data[1].Modality) + assert.Equal(t, 1, converted.Data[1].Index) + assert.Equal(t, []float64{0.3, 0.4}, converted.Data[1].Embedding.EmbeddingArray) + // Nova reports its token count in a response header only, so Usage stays nil + // here and the provider backfills it from X-Amzn-Bedrock-Input-Token-Count. + assert.Nil(t, converted.Usage) + }) + + t.Run("a combined audio-video vector is labelled as such", func(t *testing.T) { + converted, err := (&BedrockNovaEmbeddingResponse{Embeddings: []BedrockNovaEmbedding{ + {Embedding: []float64{0.1}, EmbeddingType: "AUDIO_VIDEO_COMBINED"}, + }}).ToBifrostEmbeddingResponse() + require.NoError(t, err) + require.Len(t, converted.Data, 1) + assert.Equal(t, schemas.EmbeddingModalityAudioVideo, converted.Data[0].Modality) + }) + + t.Run("a label we do not know leaves the modality empty", func(t *testing.T) { + converted, err := (&BedrockNovaEmbeddingResponse{Embeddings: []BedrockNovaEmbedding{ + {Embedding: []float64{0.1}, EmbeddingType: "SOMETHING_NEW"}, + }}).ToBifrostEmbeddingResponse() + require.NoError(t, err) + require.Len(t, converted.Data, 1) + assert.Empty(t, converted.Data[0].Modality) + assert.Equal(t, []float64{0.1}, converted.Data[0].Embedding.EmbeddingArray) + }) +} + +// The Bedrock integration fronts the native invoke route, so a Nova body written for +// AWS must survive the trip through the canonical schema and come back in its own +// envelope rather than Titan's or Cohere's. +func TestBedrockNovaEmbeddingInvokeRoundTrip(t *testing.T) { + const novaModel = "amazon.nova-2-multimodal-embeddings-v1:0" + + t.Run("a SINGLE_EMBEDDING body is detected as an embedding request", func(t *testing.T) { + assert.Equal(t, schemas.EmbeddingRequest, DetectInvokeRequestType( + []byte(`{"taskType":"SINGLE_EMBEDDING","singleEmbeddingParams":{"embeddingPurpose":"GENERIC_INDEX","text":{"truncationMode":"END","value":"hello"}}}`), + novaModel)) + }) + + t.Run("the native text body round-trips unchanged", func(t *testing.T) { + var invokeRequest BedrockInvokeRequest + require.NoError(t, json.Unmarshal([]byte(`{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "DOCUMENT_RETRIEVAL", + "embeddingDimension": 1024, + "text": {"truncationMode": "START", "value": "alpha kestrel harbor"} + } + }`), &invokeRequest)) + invokeRequest.ModelID = "bedrock/" + novaModel + + converted, err := invokeRequest.ToBifrostEmbeddingRequest(testBedrockCtx()) + require.NoError(t, err) + require.Len(t, converted.Input, 1) + require.NotNil(t, converted.Params.Dimensions) + assert.Equal(t, 1024, *converted.Params.Dimensions) + + outbound, err := ToBedrockNovaEmbeddingRequest(converted) + require.NoError(t, err) + wireBody, err := json.Marshal(outbound) + require.NoError(t, err) + assert.JSONEq(t, `{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "DOCUMENT_RETRIEVAL", + "embeddingDimension": 1024, + "text": {"truncationMode": "START", "value": "alpha kestrel harbor"} + } + }`, string(wireBody)) + }) + + t.Run("an s3 video body keeps its source and embeddingMode", func(t *testing.T) { + var invokeRequest BedrockInvokeRequest + require.NoError(t, json.Unmarshal([]byte(`{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "GENERIC_INDEX", + "video": { + "format": "mp4", + "embeddingMode": "AUDIO_VIDEO_SEPARATE", + "source": {"s3Location": {"uri": "s3://my-bucket/clips/counter.mp4"}} + } + } + }`), &invokeRequest)) + invokeRequest.ModelID = "bedrock/" + novaModel + + converted, err := invokeRequest.ToBifrostEmbeddingRequest(testBedrockCtx()) + require.NoError(t, err) + outbound, err := ToBedrockNovaEmbeddingRequest(converted) + require.NoError(t, err) + require.NotNil(t, outbound.SingleEmbeddingParams.Video) + assert.Equal(t, "mp4", outbound.SingleEmbeddingParams.Video.Format) + assert.Equal(t, BedrockNovaVideoMode("AUDIO_VIDEO_SEPARATE"), outbound.SingleEmbeddingParams.Video.EmbeddingMode) + require.NotNil(t, outbound.SingleEmbeddingParams.Video.Source.S3Location) + assert.Equal(t, "s3://my-bucket/clips/counter.mp4", outbound.SingleEmbeddingParams.Video.Source.S3Location.URI) + }) + + t.Run("the response comes back in Nova's envelope, not Titan's", func(t *testing.T) { + native, err := ToBedrockEmbeddingInvokeResponse(testBedrockCtx(), &schemas.BifrostEmbeddingResponse{ + Model: novaModel, + Data: []schemas.EmbeddingData{ + {Index: 0, Object: "embedding", Modality: schemas.EmbeddingModalityAudio, + Embedding: schemas.EmbeddingStruct{EmbeddingArray: []float64{0.1, 0.2}}}, + {Index: 1, Object: "embedding", Modality: schemas.EmbeddingModalityVideo, + Embedding: schemas.EmbeddingStruct{EmbeddingArray: []float64{0.3, 0.4}}}, + }, + }) + require.NoError(t, err) + wireBody, err := json.Marshal(native) + require.NoError(t, err) + assert.JSONEq(t, `{"embeddings":[ + {"embedding":[0.1,0.2],"embeddingType":"AUDIO"}, + {"embedding":[0.3,0.4],"embeddingType":"VIDEO"} + ]}`, string(wireBody)) + }) + + t.Run("a text source Bifrost cannot carry is a caller error", func(t *testing.T) { + var invokeRequest BedrockInvokeRequest + require.NoError(t, json.Unmarshal([]byte(`{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "GENERIC_INDEX", + "text": {"truncationMode": "END", "source": {"s3Location": {"uri": "s3://my-bucket/doc.txt"}}} + } + }`), &invokeRequest)) + invokeRequest.ModelID = "bedrock/" + novaModel + + _, err := invokeRequest.ToBifrostEmbeddingRequest(testBedrockCtx()) + require.Error(t, err) + assert.True(t, providerUtils.IsInvalidRequestError(err)) + }) +} + +func TestDetermineEmbeddingModelType(t *testing.T) { + ctx := testBedrockCtx() + tests := []struct { + model string + modelType string + }{ + {"amazon.nova-2-multimodal-embeddings-v1:0", "nova"}, + {"amazon.titan-embed-text-v2:0", "titan"}, + {"amazon.titan-embed-image-v1", "titan"}, + {"cohere.embed-v4:0", "cohere"}, + } + for _, test := range tests { + t.Run(test.model, func(t *testing.T) { + modelType, err := DetermineEmbeddingModelType(ctx, test.model) + require.NoError(t, err) + assert.Equal(t, test.modelType, modelType) + }) + } + + _, err := DetermineEmbeddingModelType(ctx, "meta.llama3-8b-instruct-v1:0") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported embedding model") +} + +// An empty text value is a 400 here rather than an AWS minLength complaint, and two of +// them are still one modality - the count must not confuse "empty" with "absent". +func TestToBedrockNovaEmbeddingRequestRejectsEmptyText(t *testing.T) { + for _, name := range []string{"one empty text part", "two empty text parts"} { + t.Run(name, func(t *testing.T) { + content := schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeText, Text: schemas.Ptr("")}} + if name == "two empty text parts" { + content = append(content, schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeText, Text: schemas.Ptr("")}) + } + _, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: "amazon.nova-2-multimodal-embeddings-v1:0", + Input: []schemas.EmbeddingInputItem{{Content: content}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "text part carries no text") + assert.True(t, providerUtils.IsInvalidRequestError(err)) + }) + } +} + +// detailLevel selects the resolution Nova reads an image at: DOCUMENT_IMAGE is the +// higher one, meant for text on a page. It is image-only - AWS rejects the key on an +// audio part - and AWS defaults it, so an absent value must stay off the wire. +func TestToBedrockNovaEmbeddingRequestDetailLevel(t *testing.T) { + const novaModel = "amazon.nova-2-multimodal-embeddings-v1:0" + imagePart := schemas.EmbeddingContentPart{Type: schemas.EmbeddingContentPartTypeImage, + Image: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==")}} + + t.Run("absent by default", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{imagePart}}}, + }) + require.NoError(t, err) + assert.Nil(t, req.SingleEmbeddingParams.Image.DetailLevel) + wireBody, err := json.Marshal(req) + require.NoError(t, err) + assert.NotContains(t, string(wireBody), "detailLevel") + }) + + t.Run("forwarded onto the image part", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{imagePart}}}, + Params: &schemas.EmbeddingParameters{ExtraParams: map[string]interface{}{"detailLevel": "DOCUMENT_IMAGE"}}, + }) + require.NoError(t, err) + require.NotNil(t, req.SingleEmbeddingParams.Image.DetailLevel) + assert.Equal(t, BedrockNovaDetailLevel("DOCUMENT_IMAGE"), *req.SingleEmbeddingParams.Image.DetailLevel) + assert.NotContains(t, req.ExtraParams, "detailLevel", "consumed, not also forwarded at the top level") + }) + + t.Run("never lands on an audio part, which AWS would refuse", func(t *testing.T) { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeAudio, + Audio: &schemas.EmbeddingMediaPart{Data: schemas.Ptr("data:audio/mpeg;base64,AA==")}}, + }}}, + Params: &schemas.EmbeddingParameters{ExtraParams: map[string]interface{}{"detailLevel": "DOCUMENT_IMAGE"}}, + }) + require.NoError(t, err) + wireBody, err := json.Marshal(req) + require.NoError(t, err) + assert.NotContains(t, string(wireBody), "detailLevel") + }) + + t.Run("a native detailLevel survives the invoke round trip", func(t *testing.T) { + var invokeRequest BedrockInvokeRequest + require.NoError(t, json.Unmarshal([]byte(`{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "DOCUMENT_RETRIEVAL", + "image": {"format": "png", "detailLevel": "DOCUMENT_IMAGE", "source": {"bytes": "iVBORw0KGgoAAAANSUhEUg=="}} + } + }`), &invokeRequest)) + invokeRequest.ModelID = "bedrock/" + novaModel + + converted, err := invokeRequest.ToBifrostEmbeddingRequest(testBedrockCtx()) + require.NoError(t, err) + outbound, err := ToBedrockNovaEmbeddingRequest(converted) + require.NoError(t, err) + wireBody, err := json.Marshal(outbound) + require.NoError(t, err) + assert.JSONEq(t, `{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "DOCUMENT_RETRIEVAL", + "image": {"format": "png", "detailLevel": "DOCUMENT_IMAGE", "source": {"bytes": "iVBORw0KGgoAAAANSUhEUg=="}} + } + }`, string(wireBody)) + }) +} + +// AWS's own reference documents video.format "3gp", which the API rejects in favour of +// "three_gp" (and accepts "mpg" as a synonym for "mpeg"). A native body copied from the +// reference must normalize to the spelling that works, and nothing may ever put "3gp" on +// the wire. +func TestToBedrockNovaEmbeddingRequestNormalizesVideoFormatSpelling(t *testing.T) { + const novaModel = "amazon.nova-2-multimodal-embeddings-v1:0" + for _, test := range []struct{ native, wire string }{ + {"3gp", "three_gp"}, + {"three_gp", "three_gp"}, + {"mpg", "mpeg"}, + {"mpeg", "mpeg"}, + } { + t.Run(test.native, func(t *testing.T) { + var invokeRequest BedrockInvokeRequest + require.NoError(t, json.Unmarshal([]byte(`{ + "taskType": "SINGLE_EMBEDDING", + "singleEmbeddingParams": { + "embeddingPurpose": "GENERIC_INDEX", + "video": {"format": "`+test.native+`", "embeddingMode": "AUDIO_VIDEO_COMBINED", "source": {"bytes": "AA=="}} + } + }`), &invokeRequest)) + invokeRequest.ModelID = "bedrock/" + novaModel + + converted, err := invokeRequest.ToBifrostEmbeddingRequest(testBedrockCtx()) + require.NoError(t, err) + outbound, err := ToBedrockNovaEmbeddingRequest(converted) + require.NoError(t, err) + assert.Equal(t, test.wire, outbound.SingleEmbeddingParams.Video.Format) + }) + } + + // The canonical path derives the format itself, so a gs-style extension or a media + // type both land on three_gp too. + for _, media := range []*schemas.EmbeddingMediaPart{ + {URL: schemas.Ptr("s3://my-bucket/clips/clip.3gp")}, + {Data: schemas.Ptr("data:video/3gpp;base64,AA==")}, + } { + req, err := ToBedrockNovaEmbeddingRequest(&schemas.BifrostEmbeddingRequest{ + Model: novaModel, + Input: []schemas.EmbeddingInputItem{{Content: schemas.EmbeddingContent{ + {Type: schemas.EmbeddingContentPartTypeVideo, Video: media}, + }}}, + }) + require.NoError(t, err) + assert.Equal(t, "three_gp", req.SingleEmbeddingParams.Video.Format) + } +} diff --git a/core/providers/bedrock/invoke.go b/core/providers/bedrock/invoke.go index cdd509d91d3..746a584bf0f 100644 --- a/core/providers/bedrock/invoke.go +++ b/core/providers/bedrock/invoke.go @@ -339,6 +339,8 @@ func DetectInvokeRequestType(body []byte, modelID string) schemas.RequestType { return schemas.ImageVariationRequest case TaskTypeInpainting, TaskTypeOutpainting, TaskTypeBackgroundRemoval: return schemas.ImageEditRequest + case TaskTypeSingleEmbedding: + return schemas.EmbeddingRequest } } @@ -540,8 +542,64 @@ func (r *BedrockInvokeRequest) ToBifrostTextCompletionRequest(ctx *schemas.Bifro return textReq.ToBifrostTextCompletionRequest(ctx) } +// novaSingleEmbeddingMediaPart maps one of Nova's native media objects back onto a +// canonical media part. The format travels as a media type, which the outbound converter +// turns into the same format name again. +func novaSingleEmbeddingMediaPart(kind, format string, source BedrockNovaEmbeddingSource) (*schemas.EmbeddingMediaPart, error) { + media := &schemas.EmbeddingMediaPart{} + if mediaType, ok := novaEmbeddingMediaTypes[format]; ok { + media.MIMEType = &mediaType + } + switch { + case source.Bytes != nil: + bytesCopy := *source.Bytes + media.Data = &bytesCopy + case source.S3Location != nil: + uri := source.S3Location.URI + media.URL = &uri + default: + return nil, providerUtils.InvalidRequestErrorf("singleEmbeddingParams.%s.source carries neither bytes nor s3Location", kind) + } + return media, nil +} + +// novaSingleEmbeddingParamsToContent maps Nova's one-modality params object onto the +// canonical content parts. +func novaSingleEmbeddingParamsToContent(params *BedrockNovaSingleEmbeddingParams) (schemas.EmbeddingContent, error) { + switch { + case params.Text != nil: + // A text source is S3-only on Nova and a canonical text part holds a string, so + // there is nowhere to put it; the value form is the one that round-trips. + if params.Text.Value == nil { + return nil, providerUtils.InvalidRequestErrorf("singleEmbeddingParams.text requires value; an S3 text source is not supported here") + } + value := *params.Text.Value + return schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeText, Text: &value}}, nil + case params.Image != nil: + media, err := novaSingleEmbeddingMediaPart("image", params.Image.Format, params.Image.Source) + if err != nil { + return nil, err + } + return schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeImage, Image: media}}, nil + case params.Audio != nil: + media, err := novaSingleEmbeddingMediaPart("audio", params.Audio.Format, params.Audio.Source) + if err != nil { + return nil, err + } + return schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeAudio, Audio: media}}, nil + case params.Video != nil: + media, err := novaSingleEmbeddingMediaPart("video", params.Video.Format, params.Video.Source) + if err != nil { + return nil, err + } + return schemas.EmbeddingContent{{Type: schemas.EmbeddingContentPartTypeVideo, Video: media}}, nil + default: + return nil, providerUtils.InvalidRequestErrorf("singleEmbeddingParams carries no text, image, audio or video") + } +} + // ToBifrostEmbeddingRequest converts the invoke request to a BifrostEmbeddingRequest. -// Handles both Titan (inputText) and Cohere (texts) embedding formats. +// Handles the Titan (inputText), Cohere (texts) and Nova (singleEmbeddingParams) formats. func (r *BedrockInvokeRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostContext) (*schemas.BifrostEmbeddingRequest, error) { modelID := r.ModelID if unescaped, err := url.PathUnescape(r.ModelID); err == nil { @@ -554,7 +612,13 @@ func (r *BedrockInvokeRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostCon } var contents []schemas.EmbeddingInputItem - if r.InputText != "" || r.InputImage != "" { + if r.SingleEmbeddingParams != nil { + content, err := novaSingleEmbeddingParamsToContent(r.SingleEmbeddingParams) + if err != nil { + return nil, err + } + contents = append(contents, schemas.EmbeddingInputItem{Content: content}) + } else if r.InputText != "" || r.InputImage != "" { // Titan's single-item shape: inputText and inputImage describe one input, so they // aggregate into one content list rather than two separate items. var content schemas.EmbeddingContent @@ -635,6 +699,22 @@ func (r *BedrockInvokeRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostCon if r.MaxTokens != nil { extraParams["max_tokens"] = *r.MaxTokens } + // Nova's required knobs live inside singleEmbeddingParams, whose other fields the + // content conversion above already consumed. + if r.SingleEmbeddingParams != nil { + if r.SingleEmbeddingParams.EmbeddingPurpose != "" { + extraParams[BedrockNovaExtraParamEmbeddingPurpose] = string(r.SingleEmbeddingParams.EmbeddingPurpose) + } + if r.SingleEmbeddingParams.Text != nil && r.SingleEmbeddingParams.Text.TruncationMode != "" { + extraParams[BedrockNovaExtraParamTruncationMode] = string(r.SingleEmbeddingParams.Text.TruncationMode) + } + if r.SingleEmbeddingParams.Video != nil && r.SingleEmbeddingParams.Video.EmbeddingMode != "" { + extraParams[BedrockNovaExtraParamEmbeddingMode] = string(r.SingleEmbeddingParams.Video.EmbeddingMode) + } + if r.SingleEmbeddingParams.Image != nil && r.SingleEmbeddingParams.Image.DetailLevel != nil { + extraParams[BedrockNovaExtraParamDetailLevel] = string(*r.SingleEmbeddingParams.Image.DetailLevel) + } + } // Merge any remaining extra params from the request for k, v := range r.ExtraParams { extraParams[k] = v @@ -649,6 +729,9 @@ func (r *BedrockInvokeRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostCon if r.EmbeddingConfig != nil && r.EmbeddingConfig.OutputEmbeddingLength != nil { dimensions = r.EmbeddingConfig.OutputEmbeddingLength } + if r.SingleEmbeddingParams != nil && r.SingleEmbeddingParams.EmbeddingDimension != nil { + dimensions = r.SingleEmbeddingParams.EmbeddingDimension + } params := &schemas.EmbeddingParameters{ Dimensions: dimensions, } @@ -1318,6 +1401,12 @@ func ToBedrockEmbeddingInvokeResponse(ctx *schemas.BifrostContext, resp *schemas } } + // Nova labels every vector with the modality it came from, and its separate video + // mode returns two for one input, so neither of the other envelopes fits. + if schemas.IsNovaModelFamily(ctx, model) { + return toBedrockNovaEmbeddingInvokeResponse(resp), nil + } + // The Titan envelope holds a single input's vectors. A response covering several // inputs — reachable when the invoke route fronts a non-Titan provider — only fits // the multi-embedding envelope; forcing it into Titan's drops all but one vector. @@ -1327,6 +1416,19 @@ func ToBedrockEmbeddingInvokeResponse(ctx *schemas.BifrostContext, resp *schemas return toBedrockTitanEmbeddingInvokeResponse(resp, tokenCount), nil } +// toBedrockNovaEmbeddingInvokeResponse rebuilds the Nova invoke envelope. Nova reports +// its token count in a response header rather than the body, so nothing carries it here. +func toBedrockNovaEmbeddingInvokeResponse(resp *schemas.BifrostEmbeddingResponse) *BedrockNovaEmbeddingResponse { + out := &BedrockNovaEmbeddingResponse{Embeddings: make([]BedrockNovaEmbedding, 0, len(resp.Data))} + for _, d := range resp.Data { + out.Embeddings = append(out.Embeddings, BedrockNovaEmbedding{ + Embedding: d.Embedding.EmbeddingArray, + EmbeddingType: novaEmbeddingTypes[d.Modality], + }) + } + return out +} + // bedrockDistinctEmbeddingInputs counts the inputs a response covers. Titan's typed // responses carry several entries for one input (float plus binary), so the index — not // the entry count — decides whether the Titan envelope fits. @@ -2132,4 +2234,4 @@ func toAnthropicInvokeStreamBytes(ctx *schemas.BifrostContext, resp *schemas.Bif return nil, fmt.Errorf("failed to marshal invoke stream event: %w", err) } return [][]byte{eventBytes}, nil -} +} \ No newline at end of file diff --git a/core/providers/bedrock/types.go b/core/providers/bedrock/types.go index 77aff8013ec..b8a8f78a98a 100644 --- a/core/providers/bedrock/types.go +++ b/core/providers/bedrock/types.go @@ -33,8 +33,10 @@ const ( bedrockServiceS3 bedrockService = "s3" ) -const MinimumReasoningMaxTokens = 1 -const DefaultCompletionMaxTokens = 4096 // Only used for relative reasoning max token calculation - not passed in body by default +const ( + MinimumReasoningMaxTokens = 1 + DefaultCompletionMaxTokens = 4096 // Only used for relative reasoning max token calculation - not passed in body by default +) // ==================== REQUEST TYPES ==================== @@ -448,8 +450,10 @@ const ( BedrockSystemToolNovaCodeInterpreter BedrockSystemToolType = "nova_code_interpreter" ) -const BedrockNovaCodeInterpreterResultType = "nova_code_interpreter_result" -const BedrockNovaGroundingResultType = "nova_grounding_result" +const ( + BedrockNovaCodeInterpreterResultType = "nova_code_interpreter_result" + BedrockNovaGroundingResultType = "nova_grounding_result" +) // BedrockSystemTool represents a Nova-managed system tool type BedrockSystemTool struct { @@ -1075,11 +1079,122 @@ type BedrockCohereEmbeddingsByType struct { Ubinary [][]int32 `json:"ubinary,omitempty"` // int32 avoids []byte→base64 JSON issue } -const TaskTypeTextImage = "TEXT_IMAGE" -const TaskTypeImageVariation = "IMAGE_VARIATION" -const TaskTypeInpainting = "INPAINTING" -const TaskTypeOutpainting = "OUTPAINTING" -const TaskTypeBackgroundRemoval = "BACKGROUND_REMOVAL" +type BedrockNovaEmbeddingPurpose string + +const ( + BedrockNovaEmbeddingPurposeGenericIndex BedrockNovaEmbeddingPurpose = "GENERIC_INDEX" + BedrockNovaEmbeddingPurposeGenericRetrieval BedrockNovaEmbeddingPurpose = "GENERIC_RETRIEVAL" + BedrockNovaEmbeddingPurposeTextRetrieval BedrockNovaEmbeddingPurpose = "TEXT_RETRIEVAL" + BedrockNovaEmbeddingPurposeImageRetrieval BedrockNovaEmbeddingPurpose = "IMAGE_RETRIEVAL" + BedrockNovaEmbeddingPurposeVideoRetrieval BedrockNovaEmbeddingPurpose = "VIDEO_RETRIEVAL" + BedrockNovaEmbeddingPurposeAudioRetrieval BedrockNovaEmbeddingPurpose = "AUDIO_RETRIEVAL" + BedrockNovaEmbeddingPurposeDocumentRetrieval BedrockNovaEmbeddingPurpose = "DOCUMENT_RETRIEVAL" + BedrockNovaEmbeddingPurposeClassification BedrockNovaEmbeddingPurpose = "CLASSIFICATION" + BedrockNovaEmbeddingPurposeClustering BedrockNovaEmbeddingPurpose = "CLUSTERING" +) + +type BedrockNovaTruncationMode string + +const ( + BedrockNovaTruncationModeEnd BedrockNovaTruncationMode = "END" + BedrockNovaTruncationModeStart BedrockNovaTruncationMode = "START" + BedrockNovaTruncationModeNone BedrockNovaTruncationMode = "NONE" // NONE fails the request instead of truncating. +) + +type BedrockNovaVideoMode string + +const ( + BedrockNovaVideoModeAudioVideoCombined BedrockNovaVideoMode = "AUDIO_VIDEO_COMBINED" // returns one vector. + BedrockNovaVideoModeAudioVideoSeparate BedrockNovaVideoMode = "AUDIO_VIDEO_SEPARATE" // returns an audio and a video vector. +) + +type BedrockNovaDetailLevel string + +const ( + BedrockNovaDetailLevelStandardImage BedrockNovaDetailLevel = "STANDARD_IMAGE" // default + BedrockNovaDetailLevelDocumentImage BedrockNovaDetailLevel = "DOCUMENT_IMAGE" // reads at a higher resolution to pick up text on a page +) + +// Nova's required knobs have no field on EmbeddingParameters, so they travel as extra +// params under their AWS names. Both ends are in this package - the invoke route writes +// them, the request converter reads them - and a mismatch would drop a value in silence. +const ( + BedrockNovaExtraParamEmbeddingPurpose = "embeddingPurpose" + BedrockNovaExtraParamTruncationMode = "truncationMode" + BedrockNovaExtraParamEmbeddingMode = "embeddingMode" + BedrockNovaExtraParamDetailLevel = "detailLevel" +) + +// BedrockNovaEmbeddingRequest is the Nova multimodal embedding invoke body. +type BedrockNovaEmbeddingRequest struct { + TaskType string `json:"taskType"` + SingleEmbeddingParams *BedrockNovaSingleEmbeddingParams `json:"singleEmbeddingParams,omitempty"` + ExtraParams map[string]any `json:"-"` +} + +// GetExtraParams implements the RequestBodyWithExtraParams interface +func (req *BedrockNovaEmbeddingRequest) GetExtraParams() map[string]any { + return req.ExtraParams +} + +type BedrockNovaSingleEmbeddingParams struct { + EmbeddingPurpose BedrockNovaEmbeddingPurpose `json:"embeddingPurpose"` + EmbeddingDimension *int `json:"embeddingDimension,omitempty"` // 256, 384, 1024 or 3072 (default) + Text *BedrockNovaEmbeddingText `json:"text,omitempty"` + Image *BedrockNovaEmbeddingImage `json:"image,omitempty"` + Audio *BedrockNovaEmbeddingMedia `json:"audio,omitempty"` + Video *BedrockNovaEmbeddingVideo `json:"video,omitempty"` +} + +type BedrockNovaEmbeddingText struct { + TruncationMode BedrockNovaTruncationMode `json:"truncationMode"` + Value *string `json:"value,omitempty"` + Source *BedrockNovaEmbeddingSource `json:"source,omitempty"` +} + +type BedrockNovaEmbeddingSource struct { + Bytes *string `json:"bytes,omitempty"` + S3Location *BedrockS3Location `json:"s3Location,omitempty"` +} + +type BedrockNovaEmbeddingMedia struct { + Format string `json:"format"` // required, mp3, wav, ogg + Source BedrockNovaEmbeddingSource `json:"source"` +} + +type BedrockNovaEmbeddingImage struct { + Format string `json:"format"` // png, jpeg, webp, gif + Source BedrockNovaEmbeddingSource `json:"source"` + DetailLevel *BedrockNovaDetailLevel `json:"detailLevel,omitempty"` +} + +type BedrockNovaEmbeddingVideo struct { + Format string `json:"format"` // mp4, mov, mkv, webm, flv, mpeg, wmv, three_gp + EmbeddingMode BedrockNovaVideoMode `json:"embeddingMode"` + Source BedrockNovaEmbeddingSource `json:"source"` +} + +// BedrockNovaEmbeddingResponse is the Nova embedding response. It carries no token +// counts (those arrive in the X-Amzn-Bedrock-Input-Token-Count header). +type BedrockNovaEmbeddingResponse struct { + Embeddings []BedrockNovaEmbedding `json:"embeddings"` +} + +// BedrockNovaEmbedding is one vector plus the modality that produced it: +// TEXT, IMAGE, AUDIO, VIDEO or AUDIO_VIDEO_COMBINED. +type BedrockNovaEmbedding struct { + Embedding []float64 `json:"embedding"` + EmbeddingType string `json:"embeddingType"` +} + +const ( + TaskTypeSingleEmbedding = "SINGLE_EMBEDDING" // Nova embeddings; SEGMENTED_EMBEDDING is StartAsyncInvoke-only + TaskTypeTextImage = "TEXT_IMAGE" + TaskTypeImageVariation = "IMAGE_VARIATION" + TaskTypeInpainting = "INPAINTING" + TaskTypeOutpainting = "OUTPAINTING" + TaskTypeBackgroundRemoval = "BACKGROUND_REMOVAL" +) // BedrockImageGenerationRequest represents a Bedrock image generation request type BedrockImageGenerationRequest struct { @@ -1466,6 +1581,10 @@ type BedrockInvokeRequest struct { OutputDimension *int `json:"output_dimension,omitempty"` // Cohere embed: 256, 512, 1024, 1536 Inputs []BedrockCohereEmbeddingInput `json:"inputs,omitempty"` // Cohere embed: mixed text+image inputs + // Nova embed: taskType SINGLE_EMBEDDING selects this params object. TaskType is + // shared with the Canvas image tasks above, which is what disambiguates the two. + SingleEmbeddingParams *BedrockNovaSingleEmbeddingParams `json:"singleEmbeddingParams,omitempty"` + // ==================== INTERNAL ==================== Stream bool `json:"-"` ExtraParams map[string]interface{} `json:"-"` @@ -1499,4 +1618,4 @@ type BedrockInvokeCohereEmbeddingResp struct { type BedrockInvokeCohereTypedEmbeddingResp struct { Embeddings BedrockCohereEmbeddingsByType `json:"embeddings"` ResponseType string `json:"response_type"` -} +} \ No newline at end of file diff --git a/core/schemas/embedding.go b/core/schemas/embedding.go index c5cbcb3462d..52aba6b7a53 100644 --- a/core/schemas/embedding.go +++ b/core/schemas/embedding.go @@ -332,10 +332,11 @@ func (p *EmbeddingParameters) Clone() *EmbeddingParameters { type EmbeddingModality string const ( - EmbeddingModalityText EmbeddingModality = "text" - EmbeddingModalityImage EmbeddingModality = "image" - EmbeddingModalityVideo EmbeddingModality = "video" - EmbeddingModalityAudio EmbeddingModality = "audio" + EmbeddingModalityText EmbeddingModality = "text" + EmbeddingModalityImage EmbeddingModality = "image" + EmbeddingModalityVideo EmbeddingModality = "video" + EmbeddingModalityAudio EmbeddingModality = "audio" + EmbeddingModalityAudioVideo EmbeddingModality = "audio_video" ) // EmbeddingVideoSegment holds the time range of a video embedding segment.