Add embedding_huggingface() engine for step_sentence_embeddings()

NAMESPACE

@@ -29,17 +29,26 @@ S3method(bake,step_tokenmerge)
 S3method(bake,step_untokenize)
 S3method(bake,step_word_embeddings)
 S3method(engine_cache_key,embedding_custom)
+S3method(engine_cache_key,embedding_huggingface)
 S3method(engine_cache_key,embedding_ollama)
 S3method(engine_check,embedding_custom)
+S3method(engine_check,embedding_huggingface)
 S3method(engine_check,embedding_ollama)
 S3method(engine_embed,embedding_custom)
+S3method(engine_embed,embedding_huggingface)
 S3method(engine_embed,embedding_ollama)
 S3method(engine_label,embedding_custom)
+S3method(engine_label,embedding_huggingface)
 S3method(engine_label,embedding_ollama)
 S3method(engine_model,embedding_custom)
+S3method(engine_model,embedding_huggingface)
 S3method(engine_model,embedding_ollama)
 S3method(engine_required_pkgs,embedding_custom)
+S3method(engine_required_pkgs,embedding_huggingface)
 S3method(engine_required_pkgs,embedding_ollama)
+S3method(engine_revision,embedding_custom)
+S3method(engine_revision,embedding_huggingface)
+S3method(engine_revision,embedding_ollama)
 S3method(format,textrecipes_tokenlist)
 S3method(obj_print_footer,textrecipes_tokenlist)
 S3method(prep,step_clean_levels)
@@ -154,6 +163,7 @@ export(all_tokenized_predictors)
 export(count_functions)
 export(download_embeddings_model2vec)
 export(embedding_custom)
+export(embedding_huggingface)
 export(embedding_ollama)
 export(ngram)
 export(read_embeddings_bpemb)

NEWS.md

@@ -1,5 +1,7 @@
 # textrecipes (development version)
 
+* Added `embedding_huggingface()`, an engine for `step_sentence_embeddings()` that runs a sentence-transformers model from the Hugging Face Hub in Python through reticulate. The model is pinned with a required `revision`, and the loaded model is kept for the rest of the R session but never stored in the recipe. `tidy()` for `step_sentence_embeddings()` gained a `revision` column. (#303)
+
 * Added `read_embeddings_model2vec()` and `read_embeddings_bpemb()`, which read downloaded Model2Vec and BPEmb files into the tibble that `step_word_embeddings()` takes as `embeddings`, `download_embeddings_model2vec()` for downloading a Model2Vec model, and a vignette, "Static embeddings with Model2Vec and BPEmb", showing how to use them. (#303)
 
 * Added `embedding_ollama()`, an engine for `step_sentence_embeddings()` that gets embeddings from a running Ollama server using httr2, and a vignette, "Sentence embeddings". (#303)

R/sentence_embeddings-engines.R

@@ -81,6 +81,12 @@ engine_model <- function(engine) {
   UseMethod("engine_model")
 }
 
+# Version of the model the engine is pinned to, or NA if the engine can't pin
+# one.
+engine_revision <- function(engine) {
+  UseMethod("engine_revision")
+}
+
 # Packages needed to run the engine, in addition to textrecipes.
 engine_required_pkgs <- function(engine) {
   UseMethod("engine_required_pkgs")
@@ -115,6 +121,11 @@ engine_model.embedding_custom <- function(engine) {
   NA_character_
 }
 
+#' @export
+engine_revision.embedding_custom <- function(engine) {
+  NA_character_
+}
+
 #' @export
 engine_required_pkgs.embedding_custom <- function(engine) {
   character()