feat: Implement BQL support (as discussed in #387)

DESCRIPTION

@@ -7,7 +7,7 @@ Authors@R: c(person("Whit", "Armstrong", role = "aut"),
                     comment = c(ORCID = "0000-0001-6419-907X")),
              person("John", "Laing", role = "aut"))
 Imports: Rcpp (>= 0.11.0), utils
-Suggests: xts, zoo, data.table, simplermarkdown, tinytest
+Suggests: xts, zoo, data.table, simplermarkdown, tinytest, jsonlite
 VignetteBuilder: simplermarkdown
 LazyLoad: yes
 LinkingTo: Rcpp, BH

NAMESPACE

@@ -11,6 +11,7 @@ export("blpConnect",
        "bdh",
        "bds",
        "beqs",
+       "bql",
        "bsrch",
        "fieldSearch",
        "fieldInfo",

R/RcppExports.R

@@ -68,6 +68,10 @@ haveBlp <- function() {
     .Call(`_Rblpapi_haveBlp`)
 }
 
+bql_Impl <- function(con, expression, verbose = FALSE) {
+    .Call(`_Rblpapi_bql_Impl`, con, expression, verbose)
+}
+
 bsrch_Impl <- function(con, domain, limit, verbose = FALSE) {
     .Call(`_Rblpapi_bsrch_Impl`, con, domain, limit, verbose)
 }

R/bql.R

@@ -0,0 +1,170 @@
+
+##  Copyright (C) 2025  Whit Armstrong and Dirk Eddelbuettel and John Laing
+##
+##  This file is part of Rblpapi
+##
+##  Rblpapi is free software: you can redistribute it and/or modify
+##  it under the terms of the GNU General Public License as published by
+##  the Free Software Foundation, either version 2 of the License, or
+##  (at your option) any later version.
+##
+##  Rblpapi is distributed in the hope that it will be useful,
+##  but WITHOUT ANY WARRANTY; without even the implied warranty of
+##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+##  GNU General Public License for more details.
+##
+##  You should have received a copy of the GNU General Public License
+##  along with Rblpapi.  If not, see <http://www.gnu.org/licenses/>.
+
+
+##' This function uses the Bloomberg API to execute 'BQL' (Bloomberg
+##' Query Language) queries via the \sQuote{//blp/bqlsvc} service --
+##' the same service used by the Excel \code{=BQL()} function.
+##'
+##' The service returns one or more JSON documents. Each queried data
+##' item is self-describing: every column carries a declared type
+##' (\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE},
+##' \sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct
+##' properly-typed \code{data.frame} columns. Parsing requires the
+##' \CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the
+##' raw JSON string(s) instead, e.g. for queries whose shape the
+##' parser does not handle.
+##'
+##' Note that \sQuote{//blp/bqlsvc} is not part of the officially
+##' documented public API; it is the service behind the Excel BQL
+##' add-in and may change without notice.
+##'
+##' @title Run 'Bloomberg Query Language' (BQL) Queries
+##' @param expression A character string with the BQL query, e.g.
+##' \code{"get(px_last) for(['IBM US Equity'])"}.
+##' @param parse A boolean indicating whether the JSON response should
+##' be parsed into \code{data.frame} objects (requires the
+##' \CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If
+##' \sQuote{FALSE} the raw JSON string(s) are returned.
+##' @param simplify A boolean indicating whether a query returning a
+##' single data item should be returned directly as a \code{data.frame}
+##' instead of a list of length one, defaults to \sQuote{TRUE}.
+##' @param verbose A boolean indicating whether verbose operation is
+##' desired, defaults to \sQuote{FALSE}.
+##' @param con A connection object as created by a \code{blpConnect}
+##' call, and retrieved via the internal function
+##' \code{defaultConnection}.
+##' @return If \code{parse} is \sQuote{TRUE}, a named list of
+##' \code{data.frame} objects, one per data item in the query's
+##' \code{get()} clause (or a single \code{data.frame} if
+##' \code{simplify} is \sQuote{TRUE} and only one item was queried).
+##' Each \code{data.frame} has an \sQuote{ID} column, a value column
+##' named after the data item, and any secondary columns (such as
+##' \sQuote{DATE} or \sQuote{CURRENCY}) the service returned. If
+##' \code{parse} is \sQuote{FALSE}, a character vector of JSON
+##' documents.
+##' @author Alexander Kammerer and Dirk Eddelbuettel
+##' @examples
+##' \dontrun{
+##' con <- blpConnect()
+##' bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])")
+##' bql("get(px_last, name) for(members('INDU Index'))", simplify=FALSE)
+##' }
+bql <- function(expression,
+                parse=TRUE,
+                simplify=TRUE,
+                verbose=FALSE,
+                con=defaultConnection()) {
+
+    res <- bql_Impl(con, expression, verbose)
+    if (!parse) return(res)
+    if (!requireNamespace("jsonlite", quietly=TRUE))
+        stop("The 'jsonlite' package is required to parse BQL responses; ",
+             "install it or call bql(..., parse=FALSE) for the raw JSON.",
+             call.=FALSE)
+    .bqlParse(res, simplify=simplify)
+}
+
+## Parse one or more raw BQL JSON documents into a named list of data.frames
+.bqlParse <- function(json, simplify=TRUE) {
+    tables <- list()
+    for (doc in json) {
+        parsed <- jsonlite::fromJSON(doc, simplifyVector=FALSE)
+        .bqlCheckExceptions(parsed)
+        for (item in parsed[["results"]]) {
+            nm <- if (is.null(item[["name"]])) "" else item[["name"]]
+            msgs <- .bqlExceptionMessages(item[["responseExceptions"]])
+            if (length(msgs))
+                warning("BQL error for item '", nm, "': ",
+                        paste(msgs, collapse="; "), call.=FALSE)
+            df <- .bqlItemToDataFrame(item)
+            if (!is.null(tables[[nm]])) {        # same item split across partial responses
+                tables[[nm]] <- rbind(tables[[nm]], df)
+            } else {
+                tables[[nm]] <- df
+            }
+        }
+    }
+    if (simplify && length(tables) == 1L) return(tables[[1L]])
+    tables
+}
+
+## Raise an R error for any top-level 'responseExceptions' the service reported
+.bqlCheckExceptions <- function(parsed) {
+    msgs <- .bqlExceptionMessages(parsed[["responseExceptions"]])
+    if (length(msgs))
+        stop("BQL error: ", paste(msgs, collapse="; "), call.=FALSE)
+    invisible(NULL)
+}
+
+.bqlExceptionMessages <- function(excs) {
+    if (is.null(excs) || length(excs) == 0L) return(character())
+    vapply(excs, function(e) {
+        msg <- e[["message"]]
+        if (is.null(msg) || !nzchar(msg)) msg <- e[["internalMessage"]]
+        if (is.null(msg) || !nzchar(msg)) msg <- "unknown BQL error"
+        msg
+    }, character(1))
+}
+
+## Convert one entry of 'results' into a data.frame using the declared
+## column types; the value column is named after the data item itself
+.bqlItemToDataFrame <- function(item) {
+    cols <- list()
+    idcol <- item[["idColumn"]]
+    if (!is.null(idcol))
+        cols[[.bqlColName(idcol, "ID")]] <- .bqlColumn(idcol)
+    valcol <- item[["valuesColumn"]]
+    if (!is.null(valcol)) {
+        nm <- if (is.null(item[["name"]]) || !nzchar(item[["name"]]))
+                  .bqlColName(valcol, "VALUE") else item[["name"]]
+        cols[[nm]] <- .bqlColumn(valcol)
+    }
+    for (sec in item[["secondaryColumns"]])
+        cols[[.bqlColName(sec, "V")]] <- .bqlColumn(sec)
+    names(cols) <- make.unique(names(cols))
+    ## avoid data.frame() name mangling and rownames
+    structure(cols,
+              class="data.frame",
+              row.names=if (length(cols)) seq_along(cols[[1L]]) else integer())
+}
+
+.bqlColName <- function(col, fallback) {
+    nm <- col[["name"]]
+    if (is.null(nm) || !nzchar(nm)) fallback else nm
+}
+
+## Convert a BQL column (list with 'type' and 'values') to a typed R vector.
+## JSON null maps to NA for every type; the string placeholders "NaN" and
+## "NA" additionally map to NA for numeric columns only, as string columns
+## may legitimately contain them (e.g. the ticker of 'NA US Equity').
+.bqlColumn <- function(col) {
+    values <- col[["values"]]
+    type <- if (is.null(col[["type"]])) "STRING" else col[["type"]]
+    values <- vapply(values, function(v) {
+        if (is.null(v)) NA_character_ else as.character(v)
+    }, character(1))
+    numericNA <- function(v) { v[v %in% c("NaN", "NA", "")] <- NA_character_; v }
+    switch(type,
+           "DOUBLE"   = as.numeric(numericNA(values)),
+           "INT"      = as.integer(numericNA(values)),
+           "BOOLEAN"  = as.logical(toupper(values)),
+           "DATE"     = as.Date(substr(values, 1L, 10L)),
+           "DATETIME" = as.POSIXct(values, format="%Y-%m-%dT%H:%M:%OS", tz="UTC"),
+           values)
+}

inst/tinytest/bql/response_grouped.json

@@ -0,0 +1 @@
+{"results":{"#mv":{"name":"#mv","offsets":[0,1,2,3,4,5],"namespace":"FUNCTION_DEFAULT","source":"BQLAnalyticsEngine","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["2027.0:Technology","2028.0:Technology","2029.0:Technology","2030.0:Technology","2031.0:Technology","2032.0:Technology"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[1.5E9,2.25E9,7.5E8,3.1E9,5.0E8,1.2E9]},"secondaryColumns":[{"name":"CURRENCY_OF_ISSUE","type":"ENUM","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"MULTIPLIER","type":"DOUBLE","rank":0,"values":[1.0,1.0,1.0,1.0,1.0,1.0]},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"ORIG_IDS","type":"STRING","rank":0,"values":[null,null,null,null,"XX000001 Corp","XX000002 Corp"]},{"name":"YEAR(MATURITY())","type":"INT","rank":0,"values":[2027,2028,2029,2030,2031,2032]},{"name":"INDUSTRY_SECTOR()","type":"STRING","rank":0,"values":["Technology","Technology","Technology","Technology","Technology","Technology"]}],"partialErrorMap":null,"responseExceptions":[],"forUniverse":false,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":[{"requestIndex":0,"responseName":"#mv"}],"responseExceptions":null,"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.288","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null}

inst/tinytest/bql/response_item_error.json

@@ -0,0 +1 @@
+{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z"],"defaultDate":true}],"responseExceptions":[{"message":"Insufficient data for 'XXX US Equity'.","type":"PARTIAL","internalMessage":"Insufficient data for 'XXX US Equity'.","messageCategory":"BQL_DATA_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"DATA_UNAVAILABLE"}]}},"ordering":["px_last"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}}

inst/tinytest/bql/response_multi_item.json

@@ -0,0 +1 @@
+{"results":{"name":{"name":"name","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["International Business Machines Corp","Apple Inc"]},"secondaryColumns":[],"responseExceptions":[]},"pe_ratio":{"name":"pe_ratio","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[23.1,33.7]},"secondaryColumns":[{"name":"AS_OF_DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z"]},{"name":"PERIOD_END_DATE","type":"DATE","rank":0,"values":["2024-09-30T00:00:00Z","2024-09-28T00:00:00Z"]},{"name":"REVISION_COUNT","type":"INT","rank":0,"values":[3,5]}],"responseExceptions":[]}},"ordering":["name","pe_ratio"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}}

inst/tinytest/bql/response_px_last.json

@@ -0,0 +1 @@
+{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity","XXX US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33,254.49,"NaN"]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z",null],"defaultDate":true},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD",null]}],"partialErrorMap":{"errorIterator":null},"responseExceptions":[],"transparency":null,"forUniverse":true,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":["px_last"],"responseExceptions":[],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null}

inst/tinytest/bql/response_string_na.json

@@ -0,0 +1 @@
+{"results":{"ticker":{"name":"ticker","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","NA US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["IBM","NA","AAPL"]},"secondaryColumns":[],"responseExceptions":[]}},"ordering":["ticker"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}}

inst/tinytest/bql/response_syntax_error.json

@@ -0,0 +1 @@
+{"results":null,"ordering":null,"responseExceptions":[{"message":"Error: Unable to parse request at 'get(px_lastfor'.","type":"PARTIAL","internalMessage":"Error: Unable to parse request at 'get(px_lastfor'.","messageCategory":"BQL_SYNTAX_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"PARSER_UNABLE"}],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null}

inst/tinytest/test_bql.R

@@ -0,0 +1,109 @@
+
+# Copyright (C) 2025  Dirk Eddelbuettel, Whit Armstrong and John Laing
+#
+# This file is part of Rblpapi.
+#
+# Rblpapi is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# Rblpapi is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Rblpapi.  If not, see <http://www.gnu.org/licenses/>.
+
+library(tinytest)
+
+if (!requireNamespace("jsonlite", quietly=TRUE)) exit_file("Skipping as 'jsonlite' is missing")
+
+library(Rblpapi)
+
+.readFixture <- function(file) {
+    paste(readLines(file.path("bql", file), warn=FALSE), collapse="\n")
+}
+
+## -- offline parsing tests (no Bloomberg connection required) --------------
+
+## single-item query: one data.frame with declared column types
+res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json"))
+expect_true(inherits(res, "data.frame"), info = "single item simplifies to data.frame")
+expect_equal(dim(res), c(3L, 4L), info = "three rows, four columns")
+expect_equal(colnames(res), c("ID", "px_last", "DATE", "CURRENCY"), info = "column names")
+expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"),
+             info = "column types follow declared JSON types")
+expect_equal(res$px_last[1:2], c(229.33, 254.49), info = "numeric values")
+expect_true(is.na(res$px_last[3]), info = "string 'NaN' becomes NA")
+expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = "JSON null becomes NA")
+expect_equal(res$DATE[1], as.Date("2024-12-17"), info = "DATE conversion")
+
+## simplify=FALSE keeps the list shape
+res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json"), simplify=FALSE)
+expect_true(is.list(res) && length(res) == 1L && names(res) == "px_last",
+            info = "simplify=FALSE returns named list")
+
+## multi-item query: one data.frame per 'get' item
+res <- Rblpapi:::.bqlParse(.readFixture("response_multi_item.json"))
+expect_true(is.list(res) && !inherits(res, "data.frame"), info = "multi item returns list")
+expect_equal(names(res), c("name", "pe_ratio"), info = "list named by data item")
+expect_equal(colnames(res$name), c("ID", "name"), info = "no secondary columns")
+expect_equal(colnames(res$pe_ratio),
+             c("ID", "pe_ratio", "AS_OF_DATE", "PERIOD_END_DATE", "REVISION_COUNT"),
+             info = "secondary columns appended")
+expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = "INT maps to integer")
+expect_equal(res$name$name[2], "Apple Inc", info = "string values")
+
+## a document split across partial responses is row-bound per item
+docs <- rep(.readFixture("response_px_last.json"), 2L)
+res <- Rblpapi:::.bqlParse(docs)
+expect_equal(nrow(res), 6L, info = "partial responses row-bound")
+
+## BQL errors surface as R errors
+expect_error(Rblpapi:::.bqlParse(.readFixture("response_syntax_error.json")),
+             pattern = "Unable to parse request", info = "responseExceptions raise")
+
+## literal "NA" strings in STRING columns are preserved, not turned into NA
+res <- Rblpapi:::.bqlParse(.readFixture("response_string_na.json"))
+expect_equal(res$ticker[2], "NA", info = "literal 'NA' string value preserved")
+expect_false(anyNA(res$ticker), info = "no spurious NAs in string column")
+
+## item-level responseExceptions surface as warnings, data is kept
+expect_warning(res <- Rblpapi:::.bqlParse(.readFixture("response_item_error.json")),
+               pattern = "Insufficient data", info = "item-level exceptions warn")
+expect_equal(nrow(res), 1L, info = "partial data still returned")
+
+## grouped aggregation, e.g. let(#mv=sum(group(amt_outstanding(),
+## by=[year(maturity()), industry_sector()]));): the ID column holds
+## composite group labels, year() yields an INT column, and ORIG_IDS is
+## null for multi-security groups but set for single-security groups
+## (synthetic values; structure verified against a live response)
+res <- Rblpapi:::.bqlParse(.readFixture("response_grouped.json"))
+expect_equal(dim(res), c(6L, 8L), info = "grouped: dimensions")
+expect_equal(colnames(res),
+             c("ID", "#mv", "CURRENCY_OF_ISSUE", "MULTIPLIER", "CURRENCY",
+               "ORIG_IDS", "YEAR(MATURITY())", "INDUSTRY_SECTOR()"),
+             info = "grouped: column names")
+expect_equal(unname(sapply(res, function(x) class(x)[1])),
+             c("character", "numeric", "character", "numeric", "character",
+               "character", "integer", "character"),
+             info = "grouped: column types incl. INT from year()")
+expect_equal(res$ID[1], "2027.0:Technology", info = "grouped: composite group id")
+expect_equal(res[["#mv"]][1], 1500000000, info = "grouped: aggregated value")
+expect_equal(res[["YEAR(MATURITY())"]][1], 2027L, info = "grouped: integer year")
+expect_true(anyNA(res$ORIG_IDS) && !all(is.na(res$ORIG_IDS)),
+            info = "grouped: ORIG_IDS null for groups, set for singletons")
+expect_equal(res$CURRENCY_OF_ISSUE[1], "USD",
+             info = "grouped: undeclared types like ENUM fall back to character")
+
+## -- live test (requires a Bloomberg connection) ----------------------------
+
+.runThisTest <- Sys.getenv("RunRblpapiUnitTests") == "yes"
+if (!.runThisTest) exit_file("Skipping live BQL test")
+
+res <- bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])")
+expect_true(inherits(res, "data.frame"), info = "live query returns data.frame")
+expect_equal(nrow(res), 2L, info = "one row per security")
+expect_true(is.numeric(res$px_last), info = "px_last is numeric")