From 5956231650edf32f5e025afcf21384ef2f126cea Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Wed, 29 Jul 2026 00:24:08 +0200 Subject: [PATCH] Add bql() for Bloomberg Query Language queries via //blp/bqlsvc Implements BQL support as discussed in #387. The C++ layer sends a 'sendQuery' request to the //blp/bqlsvc service and returns the JSON document(s) the service responds with; the R layer parses them into properly-typed data.frames (one per data item in the query's get() clause) using the column types the response itself declares, with jsonlite as an optional (Suggests) dependency. bql(..., parse=FALSE) returns the raw JSON for queries whose shape the parser cannot handle. The 'NaN'/'NA' missing-value sentinels are applied to numeric columns only so STRING columns keep legitimate 'NA' values (e.g. the ticker of 'NA US Equity'). Item-level responseExceptions (type PARTIAL, accompanying usable data) surface as warnings while the data is returned; top-level exceptions raise errors. Offline unit tests cover parsing, type mapping (including undeclared types such as ENUM), NA handling, partial responses, grouped aggregations (composite group ids, INT columns, ORIG_IDS nulls), and error propagation via synthetic fixtures whose structure was verified against live //blp/bqlsvc responses; no captured Bloomberg data is included. A live test gated on RunRblpapiUnitTests exercises the service end to end. Verified live against terminal API 3.24.6.1, including all documented example queries of the polars-bloomberg package (screens, SRCH results, segments, axes, return series). Co-Authored-By: Claude Fable 5 --- DESCRIPTION | 2 +- NAMESPACE | 1 + R/RcppExports.R | 4 + R/bql.R | 170 +++++++++++++++++++ inst/tinytest/bql/response_grouped.json | 1 + inst/tinytest/bql/response_item_error.json | 1 + inst/tinytest/bql/response_multi_item.json | 1 + inst/tinytest/bql/response_px_last.json | 1 + inst/tinytest/bql/response_string_na.json | 1 + inst/tinytest/bql/response_syntax_error.json | 1 + inst/tinytest/test_bql.R | 109 ++++++++++++ man/bql.Rd | 69 ++++++++ src/RcppExports.cpp | 14 ++ src/bql.cpp | 128 ++++++++++++++ 14 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 R/bql.R create mode 100644 inst/tinytest/bql/response_grouped.json create mode 100644 inst/tinytest/bql/response_item_error.json create mode 100644 inst/tinytest/bql/response_multi_item.json create mode 100644 inst/tinytest/bql/response_px_last.json create mode 100644 inst/tinytest/bql/response_string_na.json create mode 100644 inst/tinytest/bql/response_syntax_error.json create mode 100644 inst/tinytest/test_bql.R create mode 100644 man/bql.Rd create mode 100644 src/bql.cpp diff --git a/DESCRIPTION b/DESCRIPTION index 7701fed1..dd242072 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -7,7 +7,7 @@ Authors@R: c(person("Whit", "Armstrong", role = "aut"), comment = c(ORCID = "0000-0001-6419-907X")), person("John", "Laing", role = "aut")) Imports: Rcpp (>= 0.11.0), utils -Suggests: xts, zoo, data.table, simplermarkdown, tinytest +Suggests: xts, zoo, data.table, simplermarkdown, tinytest, jsonlite VignetteBuilder: simplermarkdown LazyLoad: yes LinkingTo: Rcpp, BH diff --git a/NAMESPACE b/NAMESPACE index 97c538fd..246a8b5c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -11,6 +11,7 @@ export("blpConnect", "bdh", "bds", "beqs", + "bql", "bsrch", "fieldSearch", "fieldInfo", diff --git a/R/RcppExports.R b/R/RcppExports.R index 5f2071c9..0d5e17ac 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -68,6 +68,10 @@ haveBlp <- function() { .Call(`_Rblpapi_haveBlp`) } +bql_Impl <- function(con, expression, verbose = FALSE) { + .Call(`_Rblpapi_bql_Impl`, con, expression, verbose) +} + bsrch_Impl <- function(con, domain, limit, verbose = FALSE) { .Call(`_Rblpapi_bsrch_Impl`, con, domain, limit, verbose) } diff --git a/R/bql.R b/R/bql.R new file mode 100644 index 00000000..736238e8 --- /dev/null +++ b/R/bql.R @@ -0,0 +1,170 @@ + +## Copyright (C) 2025 Whit Armstrong and Dirk Eddelbuettel and John Laing +## +## This file is part of Rblpapi +## +## Rblpapi is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 2 of the License, or +## (at your option) any later version. +## +## Rblpapi is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Rblpapi. If not, see . + + +##' This function uses the Bloomberg API to execute 'BQL' (Bloomberg +##' Query Language) queries via the \sQuote{//blp/bqlsvc} service -- +##' the same service used by the Excel \code{=BQL()} function. +##' +##' The service returns one or more JSON documents. Each queried data +##' item is self-describing: every column carries a declared type +##' (\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, +##' \sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct +##' properly-typed \code{data.frame} columns. Parsing requires the +##' \CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the +##' raw JSON string(s) instead, e.g. for queries whose shape the +##' parser does not handle. +##' +##' Note that \sQuote{//blp/bqlsvc} is not part of the officially +##' documented public API; it is the service behind the Excel BQL +##' add-in and may change without notice. +##' +##' @title Run 'Bloomberg Query Language' (BQL) Queries +##' @param expression A character string with the BQL query, e.g. +##' \code{"get(px_last) for(['IBM US Equity'])"}. +##' @param parse A boolean indicating whether the JSON response should +##' be parsed into \code{data.frame} objects (requires the +##' \CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If +##' \sQuote{FALSE} the raw JSON string(s) are returned. +##' @param simplify A boolean indicating whether a query returning a +##' single data item should be returned directly as a \code{data.frame} +##' instead of a list of length one, defaults to \sQuote{TRUE}. +##' @param verbose A boolean indicating whether verbose operation is +##' desired, defaults to \sQuote{FALSE}. +##' @param con A connection object as created by a \code{blpConnect} +##' call, and retrieved via the internal function +##' \code{defaultConnection}. +##' @return If \code{parse} is \sQuote{TRUE}, a named list of +##' \code{data.frame} objects, one per data item in the query's +##' \code{get()} clause (or a single \code{data.frame} if +##' \code{simplify} is \sQuote{TRUE} and only one item was queried). +##' Each \code{data.frame} has an \sQuote{ID} column, a value column +##' named after the data item, and any secondary columns (such as +##' \sQuote{DATE} or \sQuote{CURRENCY}) the service returned. If +##' \code{parse} is \sQuote{FALSE}, a character vector of JSON +##' documents. +##' @author Alexander Kammerer and Dirk Eddelbuettel +##' @examples +##' \dontrun{ +##' con <- blpConnect() +##' bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +##' bql("get(px_last, name) for(members('INDU Index'))", simplify=FALSE) +##' } +bql <- function(expression, + parse=TRUE, + simplify=TRUE, + verbose=FALSE, + con=defaultConnection()) { + + res <- bql_Impl(con, expression, verbose) + if (!parse) return(res) + if (!requireNamespace("jsonlite", quietly=TRUE)) + stop("The 'jsonlite' package is required to parse BQL responses; ", + "install it or call bql(..., parse=FALSE) for the raw JSON.", + call.=FALSE) + .bqlParse(res, simplify=simplify) +} + +## Parse one or more raw BQL JSON documents into a named list of data.frames +.bqlParse <- function(json, simplify=TRUE) { + tables <- list() + for (doc in json) { + parsed <- jsonlite::fromJSON(doc, simplifyVector=FALSE) + .bqlCheckExceptions(parsed) + for (item in parsed[["results"]]) { + nm <- if (is.null(item[["name"]])) "" else item[["name"]] + msgs <- .bqlExceptionMessages(item[["responseExceptions"]]) + if (length(msgs)) + warning("BQL error for item '", nm, "': ", + paste(msgs, collapse="; "), call.=FALSE) + df <- .bqlItemToDataFrame(item) + if (!is.null(tables[[nm]])) { # same item split across partial responses + tables[[nm]] <- rbind(tables[[nm]], df) + } else { + tables[[nm]] <- df + } + } + } + if (simplify && length(tables) == 1L) return(tables[[1L]]) + tables +} + +## Raise an R error for any top-level 'responseExceptions' the service reported +.bqlCheckExceptions <- function(parsed) { + msgs <- .bqlExceptionMessages(parsed[["responseExceptions"]]) + if (length(msgs)) + stop("BQL error: ", paste(msgs, collapse="; "), call.=FALSE) + invisible(NULL) +} + +.bqlExceptionMessages <- function(excs) { + if (is.null(excs) || length(excs) == 0L) return(character()) + vapply(excs, function(e) { + msg <- e[["message"]] + if (is.null(msg) || !nzchar(msg)) msg <- e[["internalMessage"]] + if (is.null(msg) || !nzchar(msg)) msg <- "unknown BQL error" + msg + }, character(1)) +} + +## Convert one entry of 'results' into a data.frame using the declared +## column types; the value column is named after the data item itself +.bqlItemToDataFrame <- function(item) { + cols <- list() + idcol <- item[["idColumn"]] + if (!is.null(idcol)) + cols[[.bqlColName(idcol, "ID")]] <- .bqlColumn(idcol) + valcol <- item[["valuesColumn"]] + if (!is.null(valcol)) { + nm <- if (is.null(item[["name"]]) || !nzchar(item[["name"]])) + .bqlColName(valcol, "VALUE") else item[["name"]] + cols[[nm]] <- .bqlColumn(valcol) + } + for (sec in item[["secondaryColumns"]]) + cols[[.bqlColName(sec, "V")]] <- .bqlColumn(sec) + names(cols) <- make.unique(names(cols)) + ## avoid data.frame() name mangling and rownames + structure(cols, + class="data.frame", + row.names=if (length(cols)) seq_along(cols[[1L]]) else integer()) +} + +.bqlColName <- function(col, fallback) { + nm <- col[["name"]] + if (is.null(nm) || !nzchar(nm)) fallback else nm +} + +## Convert a BQL column (list with 'type' and 'values') to a typed R vector. +## JSON null maps to NA for every type; the string placeholders "NaN" and +## "NA" additionally map to NA for numeric columns only, as string columns +## may legitimately contain them (e.g. the ticker of 'NA US Equity'). +.bqlColumn <- function(col) { + values <- col[["values"]] + type <- if (is.null(col[["type"]])) "STRING" else col[["type"]] + values <- vapply(values, function(v) { + if (is.null(v)) NA_character_ else as.character(v) + }, character(1)) + numericNA <- function(v) { v[v %in% c("NaN", "NA", "")] <- NA_character_; v } + switch(type, + "DOUBLE" = as.numeric(numericNA(values)), + "INT" = as.integer(numericNA(values)), + "BOOLEAN" = as.logical(toupper(values)), + "DATE" = as.Date(substr(values, 1L, 10L)), + "DATETIME" = as.POSIXct(values, format="%Y-%m-%dT%H:%M:%OS", tz="UTC"), + values) +} diff --git a/inst/tinytest/bql/response_grouped.json b/inst/tinytest/bql/response_grouped.json new file mode 100644 index 00000000..729e555c --- /dev/null +++ b/inst/tinytest/bql/response_grouped.json @@ -0,0 +1 @@ +{"results":{"#mv":{"name":"#mv","offsets":[0,1,2,3,4,5],"namespace":"FUNCTION_DEFAULT","source":"BQLAnalyticsEngine","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["2027.0:Technology","2028.0:Technology","2029.0:Technology","2030.0:Technology","2031.0:Technology","2032.0:Technology"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[1.5E9,2.25E9,7.5E8,3.1E9,5.0E8,1.2E9]},"secondaryColumns":[{"name":"CURRENCY_OF_ISSUE","type":"ENUM","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"MULTIPLIER","type":"DOUBLE","rank":0,"values":[1.0,1.0,1.0,1.0,1.0,1.0]},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"ORIG_IDS","type":"STRING","rank":0,"values":[null,null,null,null,"XX000001 Corp","XX000002 Corp"]},{"name":"YEAR(MATURITY())","type":"INT","rank":0,"values":[2027,2028,2029,2030,2031,2032]},{"name":"INDUSTRY_SECTOR()","type":"STRING","rank":0,"values":["Technology","Technology","Technology","Technology","Technology","Technology"]}],"partialErrorMap":null,"responseExceptions":[],"forUniverse":false,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":[{"requestIndex":0,"responseName":"#mv"}],"responseExceptions":null,"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.288","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/bql/response_item_error.json b/inst/tinytest/bql/response_item_error.json new file mode 100644 index 00000000..947202b9 --- /dev/null +++ b/inst/tinytest/bql/response_item_error.json @@ -0,0 +1 @@ +{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z"],"defaultDate":true}],"responseExceptions":[{"message":"Insufficient data for 'XXX US Equity'.","type":"PARTIAL","internalMessage":"Insufficient data for 'XXX US Equity'.","messageCategory":"BQL_DATA_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"DATA_UNAVAILABLE"}]}},"ordering":["px_last"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_multi_item.json b/inst/tinytest/bql/response_multi_item.json new file mode 100644 index 00000000..f9e815d6 --- /dev/null +++ b/inst/tinytest/bql/response_multi_item.json @@ -0,0 +1 @@ +{"results":{"name":{"name":"name","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["International Business Machines Corp","Apple Inc"]},"secondaryColumns":[],"responseExceptions":[]},"pe_ratio":{"name":"pe_ratio","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[23.1,33.7]},"secondaryColumns":[{"name":"AS_OF_DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z"]},{"name":"PERIOD_END_DATE","type":"DATE","rank":0,"values":["2024-09-30T00:00:00Z","2024-09-28T00:00:00Z"]},{"name":"REVISION_COUNT","type":"INT","rank":0,"values":[3,5]}],"responseExceptions":[]}},"ordering":["name","pe_ratio"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_px_last.json b/inst/tinytest/bql/response_px_last.json new file mode 100644 index 00000000..0e3afdd8 --- /dev/null +++ b/inst/tinytest/bql/response_px_last.json @@ -0,0 +1 @@ +{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity","XXX US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33,254.49,"NaN"]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z",null],"defaultDate":true},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD",null]}],"partialErrorMap":{"errorIterator":null},"responseExceptions":[],"transparency":null,"forUniverse":true,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":["px_last"],"responseExceptions":[],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/bql/response_string_na.json b/inst/tinytest/bql/response_string_na.json new file mode 100644 index 00000000..b08bf962 --- /dev/null +++ b/inst/tinytest/bql/response_string_na.json @@ -0,0 +1 @@ +{"results":{"ticker":{"name":"ticker","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","NA US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["IBM","NA","AAPL"]},"secondaryColumns":[],"responseExceptions":[]}},"ordering":["ticker"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_syntax_error.json b/inst/tinytest/bql/response_syntax_error.json new file mode 100644 index 00000000..3620d6a9 --- /dev/null +++ b/inst/tinytest/bql/response_syntax_error.json @@ -0,0 +1 @@ +{"results":null,"ordering":null,"responseExceptions":[{"message":"Error: Unable to parse request at 'get(px_lastfor'.","type":"PARTIAL","internalMessage":"Error: Unable to parse request at 'get(px_lastfor'.","messageCategory":"BQL_SYNTAX_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"PARSER_UNABLE"}],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R new file mode 100644 index 00000000..43945f17 --- /dev/null +++ b/inst/tinytest/test_bql.R @@ -0,0 +1,109 @@ + +# Copyright (C) 2025 Dirk Eddelbuettel, Whit Armstrong and John Laing +# +# This file is part of Rblpapi. +# +# Rblpapi is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# Rblpapi is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rblpapi. If not, see . + +library(tinytest) + +if (!requireNamespace("jsonlite", quietly=TRUE)) exit_file("Skipping as 'jsonlite' is missing") + +library(Rblpapi) + +.readFixture <- function(file) { + paste(readLines(file.path("bql", file), warn=FALSE), collapse="\n") +} + +## -- offline parsing tests (no Bloomberg connection required) -------------- + +## single-item query: one data.frame with declared column types +res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json")) +expect_true(inherits(res, "data.frame"), info = "single item simplifies to data.frame") +expect_equal(dim(res), c(3L, 4L), info = "three rows, four columns") +expect_equal(colnames(res), c("ID", "px_last", "DATE", "CURRENCY"), info = "column names") +expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"), + info = "column types follow declared JSON types") +expect_equal(res$px_last[1:2], c(229.33, 254.49), info = "numeric values") +expect_true(is.na(res$px_last[3]), info = "string 'NaN' becomes NA") +expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = "JSON null becomes NA") +expect_equal(res$DATE[1], as.Date("2024-12-17"), info = "DATE conversion") + +## simplify=FALSE keeps the list shape +res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json"), simplify=FALSE) +expect_true(is.list(res) && length(res) == 1L && names(res) == "px_last", + info = "simplify=FALSE returns named list") + +## multi-item query: one data.frame per 'get' item +res <- Rblpapi:::.bqlParse(.readFixture("response_multi_item.json")) +expect_true(is.list(res) && !inherits(res, "data.frame"), info = "multi item returns list") +expect_equal(names(res), c("name", "pe_ratio"), info = "list named by data item") +expect_equal(colnames(res$name), c("ID", "name"), info = "no secondary columns") +expect_equal(colnames(res$pe_ratio), + c("ID", "pe_ratio", "AS_OF_DATE", "PERIOD_END_DATE", "REVISION_COUNT"), + info = "secondary columns appended") +expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = "INT maps to integer") +expect_equal(res$name$name[2], "Apple Inc", info = "string values") + +## a document split across partial responses is row-bound per item +docs <- rep(.readFixture("response_px_last.json"), 2L) +res <- Rblpapi:::.bqlParse(docs) +expect_equal(nrow(res), 6L, info = "partial responses row-bound") + +## BQL errors surface as R errors +expect_error(Rblpapi:::.bqlParse(.readFixture("response_syntax_error.json")), + pattern = "Unable to parse request", info = "responseExceptions raise") + +## literal "NA" strings in STRING columns are preserved, not turned into NA +res <- Rblpapi:::.bqlParse(.readFixture("response_string_na.json")) +expect_equal(res$ticker[2], "NA", info = "literal 'NA' string value preserved") +expect_false(anyNA(res$ticker), info = "no spurious NAs in string column") + +## item-level responseExceptions surface as warnings, data is kept +expect_warning(res <- Rblpapi:::.bqlParse(.readFixture("response_item_error.json")), + pattern = "Insufficient data", info = "item-level exceptions warn") +expect_equal(nrow(res), 1L, info = "partial data still returned") + +## grouped aggregation, e.g. let(#mv=sum(group(amt_outstanding(), +## by=[year(maturity()), industry_sector()]));): the ID column holds +## composite group labels, year() yields an INT column, and ORIG_IDS is +## null for multi-security groups but set for single-security groups +## (synthetic values; structure verified against a live response) +res <- Rblpapi:::.bqlParse(.readFixture("response_grouped.json")) +expect_equal(dim(res), c(6L, 8L), info = "grouped: dimensions") +expect_equal(colnames(res), + c("ID", "#mv", "CURRENCY_OF_ISSUE", "MULTIPLIER", "CURRENCY", + "ORIG_IDS", "YEAR(MATURITY())", "INDUSTRY_SECTOR()"), + info = "grouped: column names") +expect_equal(unname(sapply(res, function(x) class(x)[1])), + c("character", "numeric", "character", "numeric", "character", + "character", "integer", "character"), + info = "grouped: column types incl. INT from year()") +expect_equal(res$ID[1], "2027.0:Technology", info = "grouped: composite group id") +expect_equal(res[["#mv"]][1], 1500000000, info = "grouped: aggregated value") +expect_equal(res[["YEAR(MATURITY())"]][1], 2027L, info = "grouped: integer year") +expect_true(anyNA(res$ORIG_IDS) && !all(is.na(res$ORIG_IDS)), + info = "grouped: ORIG_IDS null for groups, set for singletons") +expect_equal(res$CURRENCY_OF_ISSUE[1], "USD", + info = "grouped: undeclared types like ENUM fall back to character") + +## -- live test (requires a Bloomberg connection) ---------------------------- + +.runThisTest <- Sys.getenv("RunRblpapiUnitTests") == "yes" +if (!.runThisTest) exit_file("Skipping live BQL test") + +res <- bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +expect_true(inherits(res, "data.frame"), info = "live query returns data.frame") +expect_equal(nrow(res), 2L, info = "one row per security") +expect_true(is.numeric(res$px_last), info = "px_last is numeric") diff --git a/man/bql.Rd b/man/bql.Rd new file mode 100644 index 00000000..70f45ce2 --- /dev/null +++ b/man/bql.Rd @@ -0,0 +1,69 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/bql.R +\name{bql} +\alias{bql} +\title{Run 'Bloomberg Query Language' (BQL) Queries} +\usage{ +bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, + con = defaultConnection()) +} +\arguments{ +\item{expression}{A character string with the BQL query, e.g. +\code{"get(px_last) for(['IBM US Equity'])"}.} + +\item{parse}{A boolean indicating whether the JSON response should +be parsed into \code{data.frame} objects (requires the +\CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If +\sQuote{FALSE} the raw JSON string(s) are returned.} + +\item{simplify}{A boolean indicating whether a query returning a +single data item should be returned directly as a \code{data.frame} +instead of a list of length one, defaults to \sQuote{TRUE}.} + +\item{verbose}{A boolean indicating whether verbose operation is +desired, defaults to \sQuote{FALSE}.} + +\item{con}{A connection object as created by a \code{blpConnect} +call, and retrieved via the internal function +\code{defaultConnection}.} +} +\value{ +If \code{parse} is \sQuote{TRUE}, a named list of +\code{data.frame} objects, one per data item in the query's +\code{get()} clause (or a single \code{data.frame} if +\code{simplify} is \sQuote{TRUE} and only one item was queried). +Each \code{data.frame} has an \sQuote{ID} column, a value column +named after the data item, and any secondary columns (such as +\sQuote{DATE} or \sQuote{CURRENCY}) the service returned. If +\code{parse} is \sQuote{FALSE}, a character vector of JSON +documents. +} +\description{ +This function uses the Bloomberg API to execute 'BQL' (Bloomberg +Query Language) queries via the \sQuote{//blp/bqlsvc} service -- +the same service used by the Excel \code{=BQL()} function. +} +\details{ +The service returns one or more JSON documents. Each queried data +item is self-describing: every column carries a declared type +(\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, +\sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct +properly-typed \code{data.frame} columns. Parsing requires the +\CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the +raw JSON string(s) instead, e.g. for queries whose shape the +parser does not handle. + +Note that \sQuote{//blp/bqlsvc} is not part of the officially +documented public API; it is the service behind the Excel BQL +add-in and may change without notice. +} +\examples{ +\dontrun{ +con <- blpConnect() +bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +bql("get(px_last, name) for(members('INDU Index'))", simplify=FALSE) +} +} +\author{ +Alexander Kammerer and Dirk Eddelbuettel +} diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 3f40868b..b9cac11d 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -158,6 +158,19 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } +// bql_Impl +Rcpp::CharacterVector bql_Impl(SEXP con, std::string expression, bool verbose); +RcppExport SEXP _Rblpapi_bql_Impl(SEXP conSEXP, SEXP expressionSEXP, SEXP verboseSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< SEXP >::type con(conSEXP); + Rcpp::traits::input_parameter< std::string >::type expression(expressionSEXP); + Rcpp::traits::input_parameter< bool >::type verbose(verboseSEXP); + rcpp_result_gen = Rcpp::wrap(bql_Impl(con, expression, verbose)); + return rcpp_result_gen; +END_RCPP +} // bsrch_Impl Rcpp::DataFrame bsrch_Impl(SEXP con, std::string domain, std::string limit, bool verbose); RcppExport SEXP _Rblpapi_bsrch_Impl(SEXP conSEXP, SEXP domainSEXP, SEXP limitSEXP, SEXP verboseSEXP) { @@ -275,6 +288,7 @@ static const R_CallMethodDef CallEntries[] = { {"_Rblpapi_getHeaderVersion", (DL_FUNC) &_Rblpapi_getHeaderVersion, 0}, {"_Rblpapi_getRuntimeVersion", (DL_FUNC) &_Rblpapi_getRuntimeVersion, 0}, {"_Rblpapi_haveBlp", (DL_FUNC) &_Rblpapi_haveBlp, 0}, + {"_Rblpapi_bql_Impl", (DL_FUNC) &_Rblpapi_bql_Impl, 3}, {"_Rblpapi_bsrch_Impl", (DL_FUNC) &_Rblpapi_bsrch_Impl, 4}, {"_Rblpapi_fieldSearch_Impl", (DL_FUNC) &_Rblpapi_fieldSearch_Impl, 2}, {"_Rblpapi_getBars_Impl", (DL_FUNC) &_Rblpapi_getBars_Impl, 8}, diff --git a/src/bql.cpp b/src/bql.cpp new file mode 100644 index 00000000..8d83b564 --- /dev/null +++ b/src/bql.cpp @@ -0,0 +1,128 @@ +// +// bql.cpp -- "Bloomberg Query Language" query function for the BLP API +// +// Copyright (C) 2025 Whit Armstrong and Dirk Eddelbuettel and John Laing +// +// This file is part of Rblpapi +// +// Rblpapi is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 2 of the License, or +// (at your option) any later version. +// +// Rblpapi is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Rblpapi. If not, see . + +#if defined(HaveBlp) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Rcpp; + +using BloombergLP::blpapi::Session; +using BloombergLP::blpapi::Service; +using BloombergLP::blpapi::Request; +using BloombergLP::blpapi::Event; +using BloombergLP::blpapi::Element; +using BloombergLP::blpapi::Message; +using BloombergLP::blpapi::MessageIterator; +using BloombergLP::blpapi::Name; +using BloombergLP::blpapi::NotFoundException; + +// The //blp/bqlsvc service returns each response message as a single +// string-typed element holding a JSON document. Collect those strings; +// parsing is done R-side (see R/bql.R). +void processBqlEvent(Event event, std::vector& res, const bool verbose) { + MessageIterator msgIter(event); + while (msgIter.next()) { + Message msg = msgIter.message(); + if (verbose) msg.print(Rcpp::Rcout); + + Element response = msg.asElement(); + if (response.hasElement(Name{"responseError"})) { + Element err = response.getElement(Name{"responseError"}); + Rcpp::stop("Response error: " + std::string(err.getElementAsString(Name{"message"}))); + } + if (response.datatype() == BLPAPI_DATATYPE_STRING) { + res.push_back(response.getValueAsString()); + } else if (verbose) { + Rcpp::Rcout << "Skipping non-string message of type " + << msg.messageType().string() << std::endl; + } + } +} +#else +#include +#endif + +// [[Rcpp::export]] +Rcpp::CharacterVector bql_Impl(SEXP con, + std::string expression, + bool verbose=false) { +#if defined(HaveBlp) + Session* session = reinterpret_cast(checkExternalPointer(con, "blpapi::Session*")); + + const std::string bqlsvc = "//blp/bqlsvc"; + if (!session->openService(bqlsvc.c_str())) { + Rcpp::stop("Failed to open " + bqlsvc); + } + + Service bqlService = session->getService(bqlsvc.c_str()); + Request request = bqlService.createRequest("sendQuery"); + request.getElement(Name{"expression"}).setValue(expression.c_str()); + // the service expects the same client context the Excel BQL add-in sends + try { + Element clientContext = request.getElement(Name{"clientContext"}); + clientContext.setElement(Name{"appName"}, "EXCEL"); + } catch (NotFoundException& e) { + if (verbose) Rcpp::Rcout << "No 'clientContext' element in request schema" << std::endl; + } + + if (verbose) Rcpp::Rcout << "Sending Request: " << request << std::endl; + session->sendRequest(request); + + std::vector res; + + // Wait for events from Session + bool done = false; + while (!done) { + Event event = session->nextEvent(); + if (event.eventType() == Event::PARTIAL_RESPONSE) { + if (verbose) Rcpp::Rcout << "Processing Partial Response" << std::endl; + processBqlEvent(event, res, verbose); + } else if (event.eventType() == Event::RESPONSE) { + if (verbose) Rcpp::Rcout << "Processing Response" << std::endl; + processBqlEvent(event, res, verbose); + done = true; + } else { + MessageIterator msgIter(event); + while (msgIter.next()) { + Message msg = msgIter.message(); + if (event.eventType() == Event::SESSION_STATUS) { + if (msg.messageType() == "SessionTerminated" || + msg.messageType() == "SessionStartupFailure") { + done = true; + } + } + } + } + } + + return Rcpp::wrap(res); +#else // ie no Blp + return Rcpp::CharacterVector(); +#endif +}