feat: add nova multimodal embedding support for bedrock

core/providers/bedrock/bedrock.go

@@ -2193,7 +2193,7 @@ func (provider *BedrockProvider) ResponsesStream(ctx *schemas.BifrostContext, po
 }
 
 // Embedding generates embeddings for the given input text(s) using Amazon Bedrock.
-// Supports Titan and Cohere embedding models. Returns a BifrostResponse containing the embedding(s) and any error that occurred.
+// Supports Titan, Cohere and Nova embedding models. Returns a BifrostResponse containing the embedding(s) and any error that occurred.
 func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) {
 	if err := providerUtils.CheckOperationAllowed(schemas.Bedrock, provider.customProviderConfig, schemas.EmbeddingRequest); err != nil {
 		return nil, err
@@ -2240,6 +2240,19 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche
 		path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key)
 		rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model)
 
+	case "nova":
+		jsonData, bifrostError = providerUtils.CheckContextAndGetRequestBody(
+			ctx,
+			request,
+			func() (providerUtils.RequestBodyWithExtraParams, error) {
+				return ToBedrockNovaEmbeddingRequest(request)
+			})
+		if bifrostError != nil {
+			return nil, bifrostError
+		}
+		path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key)
+		rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model)
+
 	default:
 		return nil, providerUtils.NewConfigurationError("unsupported embedding model type")
 	}
@@ -2302,6 +2315,35 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche
 		}
 		bifrostResponse = converted
 		bifrostResponse.Model = request.Model
+
+	case "nova":
+		var novaResp BedrockNovaEmbeddingResponse
+		novaParseTracer, novaParseHandle := providerUtils.StartResponseParseSpan(ctx)
+		umErr := sonic.Unmarshal(rawResponse, &novaResp)
+		if novaParseTracer != nil {
+			if umErr != nil {
+				novaParseTracer.EndSpan(novaParseHandle, schemas.SpanStatusError, umErr.Error())
+			} else {
+				novaParseTracer.EndSpan(novaParseHandle, schemas.SpanStatusOk, "")
+			}
+		}
+		if umErr != nil {
+			return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError("error parsing Nova embedding response", umErr), jsonData, rawResponse, provider.sendBackRawRequest, provider.sendBackRawResponse, latency)
+		}
+		novaConvTracer, novaConvHandle := providerUtils.StartResponseConvertorSpan(ctx)
+		converted, convErr := novaResp.ToBifrostEmbeddingResponse()
+		if novaConvTracer != nil {
+			if convErr != nil {
+				novaConvTracer.EndSpan(novaConvHandle, schemas.SpanStatusError, convErr.Error())
+			} else {
+				novaConvTracer.EndSpan(novaConvHandle, schemas.SpanStatusOk, "")
+			}
+		}
+		if convErr != nil {
+			return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError("error parsing Nova embedding response", convErr), jsonData, rawResponse, provider.sendBackRawRequest, provider.sendBackRawResponse, latency)
+		}
+		bifrostResponse = converted
+		bifrostResponse.Model = request.Model
 	}
 
 	// Titan admits only float/binary and Cohere v3 has no base64 at all, so a base64 request

core/providers/bedrock/embedding.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"mime"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -371,13 +373,300 @@ func ToBedrockCohereEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest
 	return req, nil
 }
 
+// novaEmbeddingFormats maps the media types and file extensions a caller is likely to
+// carry onto Nova's per-modality format enum. AWS checks the declared format against
+// the bytes it detects, so an underivable format is refused here rather than guessed.
+var novaEmbeddingFormats = map[schemas.EmbeddingContentPartType]map[string]string{
+	schemas.EmbeddingContentPartTypeImage: {
+		"image/png": "png", "png": "png",
+		"image/jpeg": "jpeg", "image/jpg": "jpeg", "jpeg": "jpeg", "jpg": "jpeg",
+		"image/webp": "webp", "webp": "webp",
+		"image/gif": "gif", "gif": "gif",
+	},
+	schemas.EmbeddingContentPartTypeAudio: {
+		"audio/mpeg": "mp3", "audio/mp3": "mp3", "mp3": "mp3",
+		"audio/wav": "wav", "audio/x-wav": "wav", "audio/wave": "wav", "wav": "wav",
+		"audio/ogg": "ogg", "ogg": "ogg",
+	},
+	schemas.EmbeddingContentPartTypeVideo: {
+		"video/mp4": "mp4", "mp4": "mp4",
+		"video/quicktime": "mov", "mov": "mov",
+		"video/x-matroska": "mkv", "mkv": "mkv",
+		"video/webm": "webm", "webm": "webm",
+		"video/x-flv": "flv", "flv": "flv",
+		"video/mpeg": "mpeg", "mpeg": "mpeg", "mpg": "mpeg",
+		"video/x-ms-wmv": "wmv", "wmv": "wmv",
+		"video/3gpp": "three_gp", "3gp": "three_gp",
+	},
+}
+
+// novaEmbeddingMediaTypes is the reverse of novaEmbeddingFormats, used by the Bedrock
+// integration to hand a native format name back as a media type. Every value here is a
+// key of the table above, so a format survives the round trip unchanged.
+//
+// mpg and 3gp are inbound spellings only. AWS's reference documents 3gp, but the API
+// rejects it and takes three_gp, so a body copied from the reference normalizes to the
+// spelling that works rather than failing format derivation on the way out.
+var novaEmbeddingMediaTypes = map[string]string{
+	"png": "image/png", "jpeg": "image/jpeg", "webp": "image/webp", "gif": "image/gif",
+	"mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg",
+	"mp4": "video/mp4", "mov": "video/quicktime", "mkv": "video/x-matroska",
+	"webm": "video/webm", "flv": "video/x-flv", "mpeg": "video/mpeg", "mpg": "video/mpeg",
+	"wmv": "video/x-ms-wmv", "three_gp": "video/3gpp", "3gp": "video/3gpp",
+}
+
+// novaEmbeddingFormatHints names the media types accepted per modality, for the error
+// raised when neither mime_type nor the URL says which one a part carries.
+var novaEmbeddingFormatHints = map[schemas.EmbeddingContentPartType]string{
+	schemas.EmbeddingContentPartTypeImage: "image/png, image/jpeg, image/webp or image/gif",
+	schemas.EmbeddingContentPartTypeAudio: "audio/mpeg, audio/wav or audio/ogg",
+	schemas.EmbeddingContentPartTypeVideo: "video/mp4, video/quicktime, video/x-matroska, video/webm, video/x-flv, video/mpeg, video/x-ms-wmv or video/3gpp",
+}
+
+// novaEmbeddingFormat resolves Nova's format name from the declared media type, falling
+// back to the extension of an s3:// key when no type was given.
+func novaEmbeddingFormat(partType schemas.EmbeddingContentPartType, mimeType, rawURL string) (string, error) {
+	table := novaEmbeddingFormats[partType]
+	if table == nil {
+		return "", providerUtils.InvalidRequestErrorf("amazon Nova embedding models do not support %q parts", partType)
+	}
+	if mimeType != "" {
+		if parsed, _, err := mime.ParseMediaType(mimeType); err == nil {
+			mimeType = parsed
+		}
+		if format, ok := table[strings.ToLower(strings.TrimSpace(mimeType))]; ok {
+			return format, nil
+		}
+	}
+	if rawURL != "" {
+		path := rawURL
+		if i := strings.IndexAny(path, "?#"); i >= 0 {
+			path = path[:i]
+		}
+		if format, ok := table[strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))]; ok {
+			return format, nil
+		}
+	}
+	return "", providerUtils.InvalidRequestErrorf("cannot determine the %s format for amazon Nova: pass mime_type as one of %s", partType, novaEmbeddingFormatHints[partType])
+}
+
+// novaEmbeddingMediaPart converts one media part into Nova's format and source pair.
+// s3:// travels as s3Location, which Nova resolves itself; inline data travels as bytes.
+// An http(s) URL has no home here - Nova fetches neither - so it is refused with the
+// two shapes that do work.
+func novaEmbeddingMediaPart(partType schemas.EmbeddingContentPartType, media *schemas.EmbeddingMediaPart) (string, BedrockNovaEmbeddingSource, error) {
+	var empty BedrockNovaEmbeddingSource
+	if err := media.Validate(); err != nil {
+		return "", empty, providerUtils.InvalidRequestErrorf("%s", err)
+	}
+
+	mimeType := ""
+	if media.MIMEType != nil {
+		mimeType = *media.MIMEType
+	}
+
+	if media.Data != nil {
+		data := *media.Data
+		if strings.HasPrefix(data, "data:") {
+			mediaType, _, payload, ok := schemas.ParseDataURL(data)
+			if !ok {
+				return "", empty, providerUtils.InvalidRequestErrorf("%s part carries a malformed data url", partType)
+			}
+			data = payload
+			if mimeType == "" {
+				mimeType = mediaType
+			}
+		}
+		if data == "" {
+			return "", empty, providerUtils.InvalidRequestErrorf("%s part has empty data", partType)
+		}
+		format, err := novaEmbeddingFormat(partType, mimeType, "")
+		if err != nil {
+			return "", empty, err
+		}
+		return format, BedrockNovaEmbeddingSource{Bytes: &data}, nil
+	}
+
+	url := *media.URL
+	s3Location, ok := bedrockS3LocationFromURL(url)
+	if !ok {
+		if strings.HasPrefix(url, "s3://") {
+			return "", empty, providerUtils.InvalidRequestErrorf("invalid s3:// %s reference %q: expected s3://bucket/key", partType, url)
+		}
+		return "", empty, providerUtils.InvalidRequestErrorf("amazon Nova embedding models read media from s3:// or inline base64 only, so %q cannot be used; send the %s inline or presign it into S3", url, partType)
+	}
+	format, err := novaEmbeddingFormat(partType, mimeType, url)
+	if err != nil {
+		return "", empty, err
+	}
+	return format, BedrockNovaEmbeddingSource{S3Location: s3Location}, nil
+}
+
+// ToBedrockNovaEmbeddingRequest converts a Bifrost embedding request to Bedrock Nova format.
+// SINGLE_EMBEDDING embeds one content item carrying one modality, so a batch or a mixed
+// item is refused here rather than turned into an AWS schema error naming a key the
+// caller never wrote.
+func ToBedrockNovaEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest) (*BedrockNovaEmbeddingRequest, error) {
+	if bifrostReq == nil {
+		return nil, fmt.Errorf("bifrost embedding request is nil")
+	}
+
+	if len(bifrostReq.Input) == 0 {
+		return nil, providerUtils.InvalidRequestErrorf("no input provided for Nova embedding")
+	}
+
+	if len(bifrostReq.Input) != 1 {
+		return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models support exactly one content item per request; got %d", len(bifrostReq.Input))
+	}
+
+	if err := schemas.EmbeddingInput(bifrostReq.Input).RejectPerItemParams("amazon Nova"); err != nil {
+		return nil, providerUtils.InvalidRequestErrorf("%s", err)
+	}
+
+	params := &BedrockNovaSingleEmbeddingParams{EmbeddingPurpose: BedrockNovaEmbeddingPurposeGenericIndex}
+	truncationMode := BedrockNovaTruncationModeEnd
+	videoMode := BedrockNovaVideoModeAudioVideoCombined
+	var detailLevel *BedrockNovaDetailLevel
+	extra := map[string]interface{}{}
+
+	if bifrostReq.Params != nil {
+		params.EmbeddingDimension = bifrostReq.Params.Dimensions
+		if bifrostReq.Params.TaskType != nil && *bifrostReq.Params.TaskType != "" {
+			params.EmbeddingPurpose = BedrockNovaEmbeddingPurpose(*bifrostReq.Params.TaskType)
+		}
+		for k, v := range bifrostReq.Params.ExtraParams {
+			s, isString := v.(string)
+			switch {
+			case k == BedrockNovaExtraParamEmbeddingPurpose && isString && s != "":
+				params.EmbeddingPurpose = BedrockNovaEmbeddingPurpose(s)
+			case k == BedrockNovaExtraParamTruncationMode && isString && s != "":
+				truncationMode = BedrockNovaTruncationMode(s)
+			case k == BedrockNovaExtraParamEmbeddingMode && isString && s != "":
+				videoMode = BedrockNovaVideoMode(s)
+			case k == BedrockNovaExtraParamDetailLevel && isString && s != "":
+				detailLevel = schemas.Ptr(BedrockNovaDetailLevel(s))
+			default:
+				extra[k] = v
+			}
+		}
+	}
+
+	var sb strings.Builder
+	hasText := false
+	modalities := 0
+	for _, part := range bifrostReq.Input[0].Content {
+		switch part.Type {
+		case schemas.EmbeddingContentPartTypeText:
+			if part.Text == nil {
+				return nil, providerUtils.InvalidRequestErrorf("text part carries no text")
+			}
+			if hasText {
+				sb.WriteString("\n")
+			} else {
+				hasText = true
+				modalities++
+			}
+			sb.WriteString(*part.Text)
+		case schemas.EmbeddingContentPartTypeImage, schemas.EmbeddingContentPartTypeAudio, schemas.EmbeddingContentPartTypeVideo:
+			media := part.Image
+			if part.Type == schemas.EmbeddingContentPartTypeAudio {
+				media = part.Audio
+			} else if part.Type == schemas.EmbeddingContentPartTypeVideo {
+				media = part.Video
+			}
+			format, source, err := novaEmbeddingMediaPart(part.Type, media)
+			if err != nil {
+				return nil, err
+			}
+			switch part.Type {
+			case schemas.EmbeddingContentPartTypeImage:
+				params.Image = &BedrockNovaEmbeddingImage{Format: format, Source: source, DetailLevel: detailLevel}
+			case schemas.EmbeddingContentPartTypeAudio:
+				params.Audio = &BedrockNovaEmbeddingMedia{Format: format, Source: source}
+			default:
+				params.Video = &BedrockNovaEmbeddingVideo{Format: format, EmbeddingMode: videoMode, Source: source}
+			}
+			modalities++
+		default:
+			return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models do not support %q parts", part.Type)
+		}
+	}
+
+	if modalities == 0 {
+		return nil, providerUtils.InvalidRequestErrorf("no input provided for Nova embedding")
+	}
+	if modalities > 1 {
+		return nil, providerUtils.InvalidRequestErrorf("amazon Nova embedding models embed one modality per request: text, image, audio or video, not a combination")
+	}
+
+	if hasText {
+		value := sb.String()
+		if strings.TrimSpace(value) == "" {
+			return nil, providerUtils.InvalidRequestErrorf("text part carries no text")
+		}
+		params.Text = &BedrockNovaEmbeddingText{TruncationMode: truncationMode, Value: &value}
+	}
+
+	req := &BedrockNovaEmbeddingRequest{
+		TaskType:              TaskTypeSingleEmbedding,
+		SingleEmbeddingParams: params,
+	}
+	if len(extra) > 0 {
+		req.ExtraParams = extra
+	}
+	return req, nil
+}
+
+// novaEmbeddingModalities maps Nova's per-vector embeddingType label onto the canonical
+// modality. AUDIO_VIDEO_COMBINED is one vector covering both, which the separate mode
+// splits into an AUDIO and a VIDEO entry.
+var novaEmbeddingModalities = map[string]schemas.EmbeddingModality{
+	"TEXT":                 schemas.EmbeddingModalityText,
+	"IMAGE":                schemas.EmbeddingModalityImage,
+	"AUDIO":                schemas.EmbeddingModalityAudio,
+	"VIDEO":                schemas.EmbeddingModalityVideo,
+	"AUDIO_VIDEO_COMBINED": schemas.EmbeddingModalityAudioVideo,
+}
+
+// novaEmbeddingTypes is the reverse of novaEmbeddingModalities, for rebuilding the
+// native envelope on the Bedrock integration's invoke route.
+var novaEmbeddingTypes = map[schemas.EmbeddingModality]string{
+	schemas.EmbeddingModalityText:       "TEXT",
+	schemas.EmbeddingModalityImage:      "IMAGE",
+	schemas.EmbeddingModalityAudio:      "AUDIO",
+	schemas.EmbeddingModalityVideo:      "VIDEO",
+	schemas.EmbeddingModalityAudioVideo: "AUDIO_VIDEO_COMBINED",
+}
+
+// ToBifrostEmbeddingResponse converts a Bedrock Nova embedding response to Bifrost format.
+// Usage is left unset: Nova reports input tokens only in the
+// X-Amzn-Bedrock-Input-Token-Count header, which the caller backfills from.
+func (r *BedrockNovaEmbeddingResponse) ToBifrostEmbeddingResponse() (*schemas.BifrostEmbeddingResponse, error) {
+	if r == nil {
+		return nil, fmt.Errorf("nil Bedrock Nova embedding response")
+	}
+
+	bifrostResponse := &schemas.BifrostEmbeddingResponse{Object: "list"}
+	for i, embedding := range r.Embeddings {
+		bifrostResponse.Data = append(bifrostResponse.Data, schemas.EmbeddingData{
+			Index:     i,
+			Object:    "embedding",
+			Modality:  novaEmbeddingModalities[embedding.EmbeddingType],
+			Embedding: schemas.EmbeddingStruct{EmbeddingArray: embedding.Embedding},
+		})
+	}
+	return bifrostResponse, nil
+}
+
 // DetermineEmbeddingModelType determines the embedding model type for the
 // current attempt. It consults the resolved alias family first
 // (model_family / model_name / model_id / alias key) and falls back to the
 // substring detectors against the wire model — so an alias to an opaque
 // Bedrock deployment that's tagged with the right family routes correctly.
 func DetermineEmbeddingModelType(ctx *schemas.BifrostContext, model string) (string, error) {
 	switch {
+	case schemas.IsNovaModelFamily(ctx, model):
+		return "nova", nil
 	case schemas.IsTitanModelFamily(ctx, model):
 		return "titan", nil
 	case schemas.IsCohereModelFamily(ctx, model):
@@ -481,4 +770,4 @@ func (r *BedrockCohereEmbeddingResponse) ToBifrostEmbeddingResponse() (*schemas.
 	}
 
 	return bifrostResponse, nil
-}
+}