From eafe618478a52f0af0196488a2d2bdfb8a3853f8 Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 16:00:14 -0700 Subject: [PATCH 1/8] Added function to move towards YAML --- DESCRIPTION | 3 +- NAMESPACE | 2 + R/DataModel.R | 190 +++++++++++++++++- R/QueryNamespace.R | 32 ++- R/SchemaGenerator.R | 144 +++++++++++-- .../resultsDataModelSpecification.yaml | 68 +++++++ tests/testthat/settings/testSchemaDef.yaml | 133 ++++++++++++ tests/testthat/test-QueryNamespace.R | 12 +- tests/testthat/test-SchemaGenerator.R | 4 +- tests/testthat/test-Utils.R | 4 +- tests/testthat/test-YamlNamespace.R | 119 +++++++++++ tests/testthat/test-YamlSchemaGenerator.R | 112 +++++++++++ tests/testthat/test-YamlSpecification.R | 77 +++++++ 13 files changed, 868 insertions(+), 32 deletions(-) create mode 100644 tests/testthat/settings/resultsDataModelSpecification.yaml create mode 100644 tests/testthat/settings/testSchemaDef.yaml create mode 100644 tests/testthat/test-YamlNamespace.R create mode 100644 tests/testthat/test-YamlSchemaGenerator.R create mode 100644 tests/testthat/test-YamlSpecification.R diff --git a/DESCRIPTION b/DESCRIPTION index 09dcf56..8fe333f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -30,7 +30,8 @@ Imports: rlang, lubridate, fastmap, - withr + withr, + yaml Suggests: testthat (>= 3.0.0), RSQLite, diff --git a/NAMESPACE b/NAMESPACE index 7290336..ed5c17f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -14,6 +14,8 @@ export(enablePythonUploads) export(generateSqlSchema) export(grantTablePermissions) export(install_psycopg2) +export(loadResultsDataModelFromYaml) +export(csvToYaml) export(loadResultsDataModelSpecifications) export(pyPgUploadEnabled) export(pyUploadCsv) diff --git a/R/DataModel.R b/R/DataModel.R index d709f32..9c16d09 100644 --- a/R/DataModel.R +++ b/R/DataModel.R @@ -859,20 +859,208 @@ checkSpecificationColumns <- function(columnNames) { } +.yamlColDefaults <- list( + primaryKey = "No", + isRequired = "Yes", + optional = "No", + minCellCount = "No", + emptyIsNa = "Yes", + description = NA_character_ +) + +.yamlSpecToDataFrame <- function(yamlContent) { + if (!"namespace" %in% names(yamlContent)) { + stop("YAML specification must contain a 'namespace' key") + } + + rows <- list() + + for (nsName in names(yamlContent$namespace)) { + nsDef <- yamlContent$namespace[[nsName]] + if (!"tables" %in% names(nsDef)) { + stop(sprintf("Namespace '%s' must contain a 'tables' key", nsName)) + } + + for (tableName in names(nsDef$tables)) { + tableDef <- nsDef$tables[[tableName]] + if (!"columns" %in% names(tableDef)) { + stop(sprintf("Table '%s' in namespace '%s' must contain a 'columns' key", tableName, nsName)) + } + + for (col in tableDef$columns) { + if (is.null(col$name)) { + stop(sprintf("Column in table '%s' (namespace '%s') must have a 'name'", tableName, nsName)) + } + if (is.null(col$type)) { + stop(sprintf("Column '%s' in table '%s' (namespace '%s') must have a 'type'", col$name, tableName, nsName)) + } + + row <- list( + namespace = nsName, + tableName = tableName, + columnName = col$name, + dataType = col$type, + primaryKey = ifelse(isTRUE(col$primary_key), "Yes", .yamlColDefaults$primaryKey), + isRequired = ifelse(isFALSE(col$required), "No", .yamlColDefaults$isRequired), + optional = ifelse(isTRUE(col$optional), "Yes", .yamlColDefaults$optional), + minCellCount = ifelse(isTRUE(col$min_cell_count), "Yes", .yamlColDefaults$minCellCount), + emptyIsNa = ifelse(isFALSE(col$empty_is_na), "No", .yamlColDefaults$emptyIsNa), + description = if (is.null(col$description)) .yamlColDefaults$description else col$description + ) + rows[[length(rows) + 1]] <- row + } + } + } + + spec <- dplyr::bind_rows(rows) + return(spec) +} + +#' Load results data model from YAML file +#' +#' @description +#' Load a YAML results data model specification file. Returns a list containing +#' the flattened specification data frame and optional platform-specific configuration. +#' +#' @param filePath Path to a valid YAML file +#' @return A list with elements: +#' \item{specification}{A tibble data frame with columns: namespace, tableName, columnName, dataType, primaryKey, isRequired, optional, minCellCount, emptyIsNa, description} +#' \item{platforms}{Platform-specific configuration (postgresql, sql_server, duckdb, sqlite) or NULL} +#' +#' @export +loadResultsDataModelFromYaml <- function(filePath) { + checkmate::assertFileExists(filePath) + yamlContent <- yaml::read_yaml(filePath) + spec <- .yamlSpecToDataFrame(yamlContent) + assertSpecificationColumns(colnames(spec)) + + platforms <- yamlContent$platforms %||% NULL + + list( + specification = spec, + platforms = platforms + ) +} + + #' Get specifications from a given file path -#' @param filePath path to a valid csv file +#' @param filePath path to a valid csv or yaml file #' @return #' A tibble data frame object with specifications #' #' @export loadResultsDataModelSpecifications <- function(filePath) { checkmate::assertFileExists(filePath) + if (grepl("\\.ya?ml$", filePath, ignore.case = TRUE)) { + result <- loadResultsDataModelFromYaml(filePath) + return(result$specification) + } + warning( + "CSV-based results data model specifications are deprecated. ", + "Use the namespaced YAML format instead. ", + "See the 'YAML Specification Format' vignette and the csvToYaml() function to migrate.", + call. = FALSE + ) spec <- readr::read_csv(file = filePath, col_types = readr::cols()) colnames(spec) <- SqlRender::snakeCaseToCamelCase(colnames(spec)) assertSpecificationColumns(colnames(spec)) return(spec) } +#' Convert a CSV specification to YAML format +#' +#' @description +#' Reads a CSV results data model specification file and writes the equivalent +#' YAML file. This is a migration helper to transition from the legacy CSV format +#' to the namespaced YAML format. +#' +#' @param csvFilepath Path to the CSV specification file +#' @param yamlOutputPath Path to write the YAML output file +#' @param namespace Namespace name to use for all tables in the output. +#' Defaults to "default". +#' @param overwrite Boolean - overwrite existing output file? +#' +#' @return Invisibly returns the YAML content as a string +#' @export +csvToYaml <- function(csvFilepath, + yamlOutputPath, + namespace = "default", + overwrite = FALSE) { + checkmate::assertFileExists(csvFilepath) + checkmate::assertString(namespace, min.chars = 1) + + if (file.exists(yamlOutputPath) && !overwrite) { + stop("Output file ", yamlOutputPath, " already exists. Set overwrite = TRUE to continue") + } + + spec <- loadResultsDataModelSpecifications(csvFilepath) + + tables <- list() + for (tbl in unique(spec$tableName)) { + tblSpec <- spec[spec$tableName == tbl, ] + columns <- list() + for (i in seq_len(nrow(tblSpec))) { + col <- list( + name = tblSpec$columnName[i], + type = tblSpec$dataType[i] + ) + + pk <- tolower(tblSpec$primaryKey[i]) + if (pk == "yes") { + col$primary_key <- TRUE + } + + if ("optional" %in% colnames(tblSpec)) { + opt <- tolower(tblSpec$optional[i]) + if (opt == "yes") { + col$optional <- TRUE + } + } + + if ("isRequired" %in% colnames(tblSpec)) { + req <- tolower(tblSpec$isRequired[i]) + if (req == "no") { + col$required <- FALSE + } + } + + if ("minCellCount" %in% colnames(tblSpec)) { + mcc <- tolower(tblSpec$minCellCount[i]) + if (mcc == "yes") { + col$min_cell_count <- TRUE + } + } + + if ("emptyIsNa" %in% colnames(tblSpec)) { + ein <- tolower(tblSpec$emptyIsNa[i]) + if (ein == "no") { + col$empty_is_na <- FALSE + } + } + + if ("description" %in% colnames(tblSpec) && !is.na(tblSpec$description[i])) { + col$description <- tblSpec$description[i] + } + + columns[[i]] <- col + } + + tables[[tbl]] <- list(columns = columns) + } + + yamlContent <- list( + version = "1.0", + namespace = list() + ) + yamlContent$namespace[[namespace]] <- list(tables = tables) + + yamlStr <- yaml::as.yaml(yamlContent, indent.mapping.sequence = TRUE) + + writeLines(yamlStr, yamlOutputPath) + + invisible(yamlStr) +} + #' This helper function will convert the data in the #' primary key values in the `chunk` which is read from diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R index 989d945..3a1290f 100644 --- a/R/QueryNamespace.R +++ b/R/QueryNamespace.R @@ -194,21 +194,39 @@ QueryNamespace <- R6::R6Class( #' add table specification #' @description #' add a variable to automatically be replaced in query strings (e.g. @database_schema.@table_name becomes - #' 'database_schema.table_1') - #' @param tableSpecification table specification data.frame conforming to column names tableName, columnName, dataType and primaryKey + #' 'database_schema.table_1'). If the specification data.frame contains a 'namespace' column, table names + #' will be registered as '@namespace_tableName' and map to '{tablePrefix}{namespace}_{tableName}'. + #' @param tableSpecification table specification data.frame conforming to column names tableName, columnName, dataType and primaryKey. + #' May optionally include a 'namespace' column for namespace-aware registration. #' @param useTablePrefix prefix the results with the tablePrefix (TRUE) #' @param tablePrefix prefix string - defaults to class variable set during initialization #' @param replace replace existing variables of the same name addTableSpecification = function(tableSpecification, useTablePrefix = TRUE, tablePrefix = self$tablePrefix, replace = TRUE) { checkmate::assertString(tablePrefix) assertSpecificationColumns(colnames(tableSpecification)) - for (tableName in tableSpecification$tableName |> unique()) { - replacementVar <- tableName - if (useTablePrefix) { - replacementVar <- paste0(tablePrefix, replacementVar) + + hasNamespace <- "namespace" %in% colnames(tableSpecification) + + tableEntries <- tableSpecification |> + dplyr::select(dplyr::any_of(c("namespace", "tableName"))) |> + dplyr::distinct() + + for (i in seq_len(nrow(tableEntries))) { + ns <- tableEntries$namespace[i] + tableName <- tableEntries$tableName[i] + + if (hasNamespace && !is.na(ns)) { + registryKey <- paste0(ns, "_", tableName) + replacementVar <- paste0(tablePrefix, ns, "_", tableName) + } else if (useTablePrefix) { + registryKey <- tableName + replacementVar <- paste0(tablePrefix, tableName) + } else { + registryKey <- tableName + replacementVar <- tableName } - self$addReplacementVariable(tableName, replacementVar, replace = replace) + self$addReplacementVariable(registryKey, replacementVar, replace = replace) } invisible(NULL) }, diff --git a/R/SchemaGenerator.R b/R/SchemaGenerator.R index 1c7f7af..b924877 100644 --- a/R/SchemaGenerator.R +++ b/R/SchemaGenerator.R @@ -25,38 +25,118 @@ str } +.getPlatformConfig <- function(platforms, platform, namespace, table) { + if (is.null(platforms) || is.null(platform)) { + return(NULL) + } + platforms[[platform]]$namespace[[namespace]]$tables[[table]] +} + +.generatePartitionBy <- function(partitionBy, tableName, platform) { + if (is.null(partitionBy)) { + return("") + } + + supportedPlatforms <- c("postgresql") + if (!platform %in% supportedPlatforms) { + warning(sprintf( + "Partitioning is not supported for platform '%s'. Supported platforms: %s. Skipping partitioning for table '%s'.", + platform, paste(supportedPlatforms, collapse = ", "), tableName + )) + return("") + } + + paste0(" PARTITION BY ", partitionBy) +} + +.generateIndexes <- function(platformConfig, platform) { + if (is.null(platformConfig) || is.null(platformConfig$indexes)) { + return("") + } + + supportedPlatforms <- c("postgresql", "sql_server") + if (!platform %in% supportedPlatforms) { + warning(sprintf( + "Index DDL generation is not supported for platform '%s'. Supported platforms: %s.", + platform, paste(supportedPlatforms, collapse = ", ") + )) + return("") + } + + indexStatements <- character() + for (idx in platformConfig$indexes) { + columns <- paste(idx$columns, collapse = ", ") + uniqueStr <- if (isTRUE(idx$unique)) "UNIQUE " else "" + indexName <- gsub("[^a-zA-Z0-9_]", "_", paste(c("idx", idx$columns), collapse = "_")) + indexStatements <- c(indexStatements, + sprintf("\nCREATE %sINDEX %s ON @database_schema.@table_prefix@table_name (%s);", + uniqueStr, indexName, columns)) + } + + paste(indexStatements, collapse = "") +} + #' Schema generator #' @export #' @description -#' Take a csv schema definition and create a basic sql script with it. +#' Take a csv or yaml schema definition and create a basic sql script with it. +#' For YAML files with platform-specific configuration, use the `platform` parameter +#' to include features like partitioning and indexes. #' returns string containing the sql for the table -#' @param csvFilepath Path to schema file. Csv file must have the columns: -#' "table_name", "column_name", "data_type", "primary_key" -#' @param schemaDefinition A schemaDefintiion data.frame` with the columns: -#' tableName, columnName, dataType, isRequired, primaryKey +#' @param csvFilepath Path to schema file (csv or yaml). Csv file must have the columns: +#' "table_name", "column_name", "data_type", "primary_key". +#' Yaml file must follow the namespaced YAML schema format. +#' @param schemaDefinition A schemaDefinition data.frame with the columns: +#' tableName, columnName, dataType, isRequired, primaryKey. +#' May optionally include a 'namespace' column. #' @param sqlOutputPath File to write sql to. #' @param overwrite Boolean - overwrite existing file? +#' @param platform Target database platform for platform-specific DDL +#' (e.g. "postgresql", "sql_server", "sqlite", "duckdb"). +#' Only applies when loading from a YAML file. generateSqlSchema <- function(csvFilepath = NULL, schemaDefinition = NULL, sqlOutputPath = NULL, - overwrite = FALSE) { + overwrite = FALSE, + platform = NULL) { if (all(is.null(c(csvFilepath, schemaDefinition)))) { - stop("Must spcify a csv file or schema definition") - } else if (is.null(schemaDefinition)) { + stop("Must spcify a csv or yaml file or schema definition") + } + + platformConfig <- NULL + + if (is.null(schemaDefinition)) { if (!is.null(sqlOutputPath) && (file.exists(sqlOutputPath) & !overwrite)) { stop("Output file ", sqlOutputPath, "already exists. Set overwrite = TRUE to continue") } checkmate::assertFileExists(csvFilepath) - schemaDefinition <- readr::read_csv(csvFilepath, show_col_types = FALSE) - names(schemaDefinition) <- SqlRender::snakeCaseToCamelCase(names(schemaDefinition)) + + if (grepl("\\.ya?ml$", csvFilepath, ignore.case = TRUE)) { + yamlResult <- loadResultsDataModelFromYaml(csvFilepath) + schemaDefinition <- yamlResult$specification + if (!is.null(platform)) { + platformConfig <- yamlResult$platforms + } + } else { + warning( + "CSV-based schema definitions are deprecated. ", + "Use the namespaced YAML format instead. ", + "See the 'YAML Specification Format' vignette and the csvToYaml() function to migrate.", + call. = FALSE + ) + schemaDefinition <- readr::read_csv(csvFilepath, show_col_types = FALSE) + names(schemaDefinition) <- SqlRender::snakeCaseToCamelCase(names(schemaDefinition)) + } } assertSpecificationColumns(colnames(schemaDefinition)) + hasNamespace <- "namespace" %in% colnames(schemaDefinition) + tableSqlStr <- " CREATE TABLE @database_schema.@table_prefix@table_name ( @table_columns -); +)@partition_by; " fullScript <- "" defs <- "{DEFAULT @table_prefix = ''}\n" @@ -72,18 +152,50 @@ CREATE TABLE @database_schema.@table_prefix@table_name ( } columnDefinitions <- paste(columnDefinitions, collapse = ",\n") + + varName <- table + if (hasNamespace) { + ns <- tableColumns$namespace[1] + if (!is.na(ns)) { + varName <- paste0(ns, "_", table) + } + } + + partitionBySql <- "" + if (!is.null(platformConfig) && hasNamespace) { + ns <- tableColumns$namespace[1] + if (!is.na(ns)) { + tblPlatformConfig <- .getPlatformConfig(platformConfig, platform, ns, table) + partitionBySql <- .generatePartitionBy(tblPlatformConfig$partition_by, varName, platform) + } + } + tableString <- SqlRender::render(tableSqlStr, - table_name = paste0("@", table), - table_columns = columnDefinitions + table_name = paste0("@", varName), + table_columns = columnDefinitions, + partition_by = partitionBySql ) - tableDefStr <- paste0("{DEFAULT @", table, " = ", table, "}\n") + tableDefStr <- paste0("{DEFAULT @", varName, " = ", varName, "}\n") defs <- paste0(defs, tableDefStr) - fullScript <- paste(fullScript, tableString) + indexSql <- "" + if (!is.null(platformConfig) && hasNamespace) { + ns <- tableColumns$namespace[1] + if (!is.na(ns)) { + tblPlatformConfig <- .getPlatformConfig(platformConfig, platform, ns, table) + indexSql <- .generateIndexes(tblPlatformConfig, platform) + if (nchar(indexSql) > 0) { + indexSql <- SqlRender::render(indexSql, + table_name = paste0("@", varName) + ) + } + } + } + + fullScript <- paste(fullScript, tableString, indexSql) } - # Get columns for each table lines <- paste(defs, fullScript) if (!is.null(sqlOutputPath)) { writeLines(lines, sqlOutputPath) diff --git a/tests/testthat/settings/resultsDataModelSpecification.yaml b/tests/testthat/settings/resultsDataModelSpecification.yaml new file mode 100644 index 0000000..a4ed6e1 --- /dev/null +++ b/tests/testthat/settings/resultsDataModelSpecification.yaml @@ -0,0 +1,68 @@ +version: "1.0" + +namespace: + test: + description: "Test namespace for basic upload/validation" + tables: + test_table_1: + columns: + - name: database_id + type: varchar + primary_key: true + - name: analysis_id + type: bigint + primary_key: true + - name: analysis_name + type: varchar + - name: domain_id + type: varchar(20) + optional: true + - name: start_day + type: float + optional: true + - name: end_day + type: float + optional: true + - name: is_binary + type: varchar(1) + - name: missing_means_zero + type: varchar(1) + optional: true + test_table_2: + columns: + - name: database_id + type: varchar + primary_key: true + - name: analysis2_id + type: bigint + primary_key: true + - name: concept_id + type: int + - name: logic_description + type: varchar + optional: true + - name: valid_start_date + type: Date + - name: concept_name + type: varchar(255) + - name: p_10_value + type: float + test_table_3: + columns: + - name: database_id + type: varchar + primary_key: true + - name: analysis3_id + type: bigint + primary_key: true + - name: concept_id + type: int + - name: logic_description + type: varchar + optional: true + - name: valid_start_date + type: Date + - name: concept_name + type: varchar(255) + - name: p_10_value + type: float diff --git a/tests/testthat/settings/testSchemaDef.yaml b/tests/testthat/settings/testSchemaDef.yaml new file mode 100644 index 0000000..d9807e4 --- /dev/null +++ b/tests/testthat/settings/testSchemaDef.yaml @@ -0,0 +1,133 @@ +version: "1.0" + +namespace: + cg: + description: "Cohort Generator namespace" + tables: + cohort_definition: + columns: + - name: cohort_definition_id + type: bigint + primary_key: true + - name: concept_id + type: bigint + - name: cohort_name + type: varchar + - name: short_name + type: varchar + - name: atc_flag + type: int + - name: database_id + type: bigint + primary_key: true + cdm_source_info: + columns: + - name: database_id + type: bigint + primary_key: true + - name: cdm_source_abbreviation + type: varchar + optional: true + - name: cdm_holder + type: varchar + optional: true + - name: source_description + type: varchar + optional: true + - name: source_documentation_reference + type: varchar + optional: true + - name: cdm_etl_reference + type: varchar + optional: true + - name: source_release_date + type: date + optional: true + - name: cdm_release_date + type: date + optional: true + - name: cdm_version + type: varchar + optional: true + - name: vocabulary_version + type: varchar + optional: true + cohort_counts: + columns: + - name: num_persons + type: bigint + min_cell_count: true + - name: cohort_definition_id + type: bigint + primary_key: true + - name: database_id + type: bigint + primary_key: true + cosine_similarity: + columns: + - name: database_id + type: bigint + primary_key: true + - name: cohort_definition_id_1 + type: bigint + primary_key: true + - name: cohort_definition_id_2 + type: bigint + primary_key: true + - name: covariate_type + type: varchar + - name: cosine_similarity + type: float + covariate_definition: + columns: + - name: covariate_id + type: bigint + - name: covariate_name + type: varchar + - name: concept_id + type: bigint + - name: time_at_risk_start + type: int + - name: time_at_risk_end + type: int + - name: covariate_type + type: varchar + covariate_mean: + columns: + - name: database_id + type: bigint + primary_key: true + - name: cohort_definition_id + type: bigint + primary_key: true + - name: covariate_id + type: bigint + primary_key: true + - name: covariate_mean + type: float + primary_key: true + +platforms: + postgresql: + namespace: + cg: + tables: + cohort_counts: + partition_by: RANGE (cohort_definition_id) + indexes: + - columns: [database_id, cohort_definition_id] + unique: true + covariate_mean: + partition_by: RANGE (cohort_definition_id) + sql_server: + namespace: + cg: + tables: + cohort_counts: + indexes: + - columns: [database_id] + covariate_mean: + indexes: + - columns: [cohort_definition_id, covariate_id] + duckdb: {} + sqlite: {} diff --git a/tests/testthat/test-QueryNamespace.R b/tests/testthat/test-QueryNamespace.R index ac34146..e94794a 100644 --- a/tests/testthat/test-QueryNamespace.R +++ b/tests/testthat/test-QueryNamespace.R @@ -78,7 +78,9 @@ test_that("create helper function works", { qns <- createQueryNamespace( connectionDetails = connectionDetails, usePooledConnection = FALSE, - tableSpecification = loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv"), + tableSpecification = suppressWarnings( + loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv") + ), resultModelSpecificationPath = NULL, tablePrefix = "", snakeCaseToCamelCase = TRUE, @@ -88,7 +90,7 @@ test_that("create helper function works", { vars <- qns$getVars() expect_true("databaseSchema" %in% names(vars)) - qns <- createQueryNamespace( + qns <- suppressWarnings(createQueryNamespace( connectionDetails = connectionDetails, usePooledConnection = TRUE, tableSpecification = NULL, @@ -99,7 +101,7 @@ test_that("create helper function works", { tablePrefix = "", snakeCaseToCamelCase = TRUE, databaseSchema = "main" - ) + )) vars <- qns$getVars() expect_true("cohort_counts" %in% names(vars)) @@ -123,7 +125,7 @@ test_that("create helper function works", { ) expect_error( - createQueryNamespace( + suppressWarnings(createQueryNamespace( connectionDetails = NULL, usePooledConnection = FALSE, tableSpecification = NULL, @@ -131,7 +133,7 @@ test_that("create helper function works", { tablePrefix = "", snakeCaseToCamelCase = TRUE, databaseSchema = "main" - ) + )) ) }) diff --git a/tests/testthat/test-SchemaGenerator.R b/tests/testthat/test-SchemaGenerator.R index 6fc2914..e2815ad 100644 --- a/tests/testthat/test-SchemaGenerator.R +++ b/tests/testthat/test-SchemaGenerator.R @@ -8,7 +8,9 @@ test_that("Schema gen from file", { DatabaseConnector::disconnect(connection) }) - schema <- generateSqlSchema(csvFilepath = "settings/testSchemaDef.csv", sqlOutputPath = tfile) + schema <- suppressWarnings( + generateSqlSchema(csvFilepath = "settings/testSchemaDef.csv", sqlOutputPath = tfile) + ) checkmate::expect_file_exists(tfile) schemaDetails <- readr::read_csv("settings/testSchemaDef.csv", show_col_types = FALSE) diff --git a/tests/testthat/test-Utils.R b/tests/testthat/test-Utils.R index 17bf3b8..3d2c63f 100644 --- a/tests/testthat/test-Utils.R +++ b/tests/testthat/test-Utils.R @@ -1,5 +1,7 @@ test_that("Grant permissions", { - tableSpecification <- loadResultsDataModelSpecifications("settings/testSchemaDef.csv") + tableSpecification <- suppressWarnings( + loadResultsDataModelSpecifications("settings/testSchemaDef.csv") + ) expect_error( grantTablePermissions( diff --git a/tests/testthat/test-YamlNamespace.R b/tests/testthat/test-YamlNamespace.R new file mode 100644 index 0000000..1802692 --- /dev/null +++ b/tests/testthat/test-YamlNamespace.R @@ -0,0 +1,119 @@ +test_that("QueryNamespace handles namespaced table specifications", { + connectionHandler <- ConnectionHandler$new(connectionDetails = connectionDetails) + + tableSpecification <- data.frame( + namespace = "cg", + tableName = "cohort", + columnName = c("cohort_definition_id", "cohort_name", "json", "sql"), + primaryKey = c("yes", "no", "no", "no"), + dataType = c("int", "varchar", "varchar", "varchar") + ) + + schemaSql <- generateSqlSchema(schemaDefinition = tableSpecification) + connectionHandler$executeSql(schemaSql, table_prefix = "cd_", database_schema = "main") + + qns <- QueryNamespace$new( + connectionHandler = connectionHandler, + tableSpecification = tableSpecification, + result_schema = "main", + tablePrefix = "cd_" + ) + on.exit({ + qns$closeConnection() + }) + + expect_equal(qns$render("@cg_cohort"), "cd_cg_cohort") + + sql <- "SELECT * FROM @result_schema.@cg_cohort WHERE cohort_definition_id = @cohort_id" + renderedSql <- qns$render(sql, cohort_id = 1) + expect_equal(renderedSql, "SELECT * FROM main.cd_cg_cohort WHERE cohort_definition_id = 1") +}) + +test_that("QueryNamespace backward compat without namespace", { + connectionHandler <- ConnectionHandler$new(connectionDetails = connectionDetails) + + tableSpecification <- data.frame( + tableName = "cohort2", + columnName = c("cohort_definition_id", "cohort_name", "json", "sql"), + primaryKey = c("yes", "no", "no", "no"), + dataType = c("int", "varchar", "varchar", "varchar") + ) + + schemaSql <- generateSqlSchema(schemaDefinition = tableSpecification) + connectionHandler$executeSql(schemaSql, table_prefix = "cd_", database_schema = "main") + + qns <- QueryNamespace$new( + connectionHandler = connectionHandler, + tableSpecification = tableSpecification, + result_schema = "main", + tablePrefix = "cd_" + ) + on.exit({ + qns$closeConnection() + }) + + expect_equal(qns$render("@cohort2"), "cd_cohort2") +}) + +test_that("QueryNamespace handles multiple namespaces from yaml", { + connectionHandler <- ConnectionHandler$new(connectionDetails = connectionDetails) + + spec <- loadResultsDataModelSpecifications("settings/testSchemaDef.yaml") + + schemaSql <- generateSqlSchema(schemaDefinition = spec) + connectionHandler$executeSql(schemaSql, table_prefix = "main_", database_schema = "main") + + qns <- QueryNamespace$new( + connectionHandler = connectionHandler, + tableSpecification = spec, + result_schema = "main", + tablePrefix = "main_" + ) + on.exit({ + qns$closeConnection() + }) + + expect_equal(qns$render("@cg_cohort_definition"), "main_cg_cohort_definition") + expect_equal(qns$render("@cg_cohort_counts"), "main_cg_cohort_counts") + expect_equal(qns$render("@cg_cosine_similarity"), "main_cg_cosine_similarity") +}) + +test_that("createQueryNamespace works with yaml spec files", { + skip_on_cran() + + qns <- createQueryNamespace( + connectionDetails = connectionDetails, + usePooledConnection = FALSE, + resultModelSpecificationPath = "settings/testSchemaDef.yaml", + tablePrefix = "test_", + snakeCaseToCamelCase = TRUE, + databaseSchema = "main" + ) + + vars <- qns$getVars() + expect_true("cg_cohort_definition" %in% names(vars)) + expect_true("cg_cdm_source_info" %in% names(vars)) + expect_true("cg_cohort_counts" %in% names(vars)) + expect_equal(vars[["cg_cohort_definition"]], "test_cg_cohort_definition") +}) + +test_that("createQueryNamespace merges csv and yaml specs", { + skip_on_cran() + + qns <- suppressWarnings(createQueryNamespace( + connectionDetails = connectionDetails, + usePooledConnection = FALSE, + resultModelSpecificationPath = c( + "settings/resultsDataModelSpecification.csv", + "settings/testSchemaDef.yaml" + ), + tablePrefix = "", + snakeCaseToCamelCase = TRUE, + databaseSchema = "main" + )) + + vars <- qns$getVars() + expect_true("test_table_1" %in% names(vars)) + expect_true("cg_cohort_definition" %in% names(vars)) + expect_true("cg_cohort_counts" %in% names(vars)) +}) diff --git a/tests/testthat/test-YamlSchemaGenerator.R b/tests/testthat/test-YamlSchemaGenerator.R new file mode 100644 index 0000000..915f3db --- /dev/null +++ b/tests/testthat/test-YamlSchemaGenerator.R @@ -0,0 +1,112 @@ +test_that("generateSqlSchema from yaml with platform config includes partitioning", { + tfile <- tempfile(fileext = ".sql") + on.exit(unlink(tfile)) + + schema <- generateSqlSchema( + csvFilepath = "settings/testSchemaDef.yaml", + sqlOutputPath = tfile, + platform = "postgresql" + ) + + checkmate::expect_file_exists(tfile) + checkmate::expect_string(schema) + + expect_true(grepl("PARTITION BY RANGE \\(cohort_definition_id\\)", schema)) + expect_true(grepl("CREATE UNIQUE INDEX", schema)) +}) + +test_that("generateSqlSchema from yaml with platform config includes indexes for sql_server", { + tfile <- tempfile(fileext = ".sql") + on.exit(unlink(tfile)) + + schema <- generateSqlSchema( + csvFilepath = "settings/testSchemaDef.yaml", + sqlOutputPath = tfile, + platform = "sql_server" + ) + + expect_true(grepl("CREATE INDEX", schema)) + expect_false(grepl("PARTITION BY", schema)) +}) + +test_that("generateSqlSchema from yaml warns when unsupported platform has partitioning", { + tmpYaml <- tempfile(fileext = ".yaml") + on.exit(unlink(tmpYaml)) + + yamlContent <- list( + version = "1.0", + namespace = list( + test = list( + tables = list( + t1 = list( + columns = list( + list(name = "id", type = "bigint", primary_key = TRUE) + ) + ) + ) + ) + ), + platforms = list( + duckdb = list( + namespace = list( + test = list( + tables = list( + t1 = list(partition_by = "RANGE (id)") + ) + ) + ) + ) + ) + ) + yaml::write_yaml(yamlContent, tmpYaml) + + expect_warning( + generateSqlSchema( + csvFilepath = tmpYaml, + platform = "duckdb" + ), + "Partitioning is not supported for platform" + ) +}) + +test_that("generateSqlSchema from yaml without platform parameter excludes platform features", { + schema <- generateSqlSchema(csvFilepath = "settings/testSchemaDef.yaml") + expect_false(grepl("PARTITION BY", schema)) + expect_false(grepl("CREATE.*INDEX", schema)) +}) + +test_that("generateSqlSchema from csv still works unchanged with deprecation warning", { + tfile <- tempfile() + on.exit(unlink(tfile)) + + expect_warning( + schema <- generateSqlSchema(csvFilepath = "settings/testSchemaDef.csv", sqlOutputPath = tfile), + "CSV-based schema definitions are deprecated" + ) + checkmate::expect_file_exists(tfile) + checkmate::expect_string(schema) + expect_false(grepl("PARTITION BY", schema)) +}) + +test_that("generateSqlSchema with namespaced data.frame and platform", { + testCd <- DatabaseConnector::createConnectionDetails(server = "testPlatformSchema.db", dbms = "sqlite") + connection <- DatabaseConnector::connect(testCd) + on.exit({ + unlink("testPlatformSchema.db") + DatabaseConnector::disconnect(connection) + }) + + spec <- loadResultsDataModelSpecifications("settings/testSchemaDef.yaml") + + schema <- generateSqlSchema(schemaDefinition = spec, platform = "sqlite") + expect_true(grepl("@cg_cohort_definition", schema, fixed = TRUE)) + expect_true(grepl("@cg_cohort_counts", schema, fixed = TRUE)) + + DatabaseConnector::renderTranslateExecuteSql(connection, schema, database_schema = "main") + + res <- DatabaseConnector::renderTranslateQuerySql(connection, + "SELECT * FROM @table_name", + table_name = "cg_cohort_definition" + ) + expect_true(is.data.frame(res)) +}) diff --git a/tests/testthat/test-YamlSpecification.R b/tests/testthat/test-YamlSpecification.R new file mode 100644 index 0000000..44fa2c8 --- /dev/null +++ b/tests/testthat/test-YamlSpecification.R @@ -0,0 +1,77 @@ +test_that("loadResultsDataModelSpecifications supports yaml files", { + spec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml") + expect_true("namespace" %in% colnames(spec)) + expect_true("tableName" %in% colnames(spec)) + expect_true("columnName" %in% colnames(spec)) + expect_true("dataType" %in% colnames(spec)) + expect_true("primaryKey" %in% colnames(spec)) + + expect_equal(unique(spec$namespace), "test") + expect_true("test_table_1" %in% spec$tableName) + expect_true("test_table_2" %in% spec$tableName) + expect_true("test_table_3" %in% spec$tableName) +}) + +test_that("loadResultsDataModelSpecifications still works with csv files and warns deprecation", { + expect_warning( + spec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv"), + "CSV-based results data model specifications are deprecated" + ) + expect_false("namespace" %in% colnames(spec)) + expect_true("tableName" %in% colnames(spec)) + expect_true("columnName" %in% colnames(spec)) + expect_equal(unique(spec$tableName), c("test_table_1", "test_table_2", "test_table_3")) +}) + +test_that("loadResultsDataModelFromYaml returns platform config", { + result <- loadResultsDataModelFromYaml("settings/testSchemaDef.yaml") + expect_true("specification" %in% names(result)) + expect_true("platforms" %in% names(result)) + + spec <- result$specification + expect_true("namespace" %in% colnames(spec)) + expect_equal(unique(spec$namespace), "cg") + + platforms <- result$platforms + expect_true("postgresql" %in% names(platforms)) + expect_true("sql_server" %in% names(platforms)) + expect_true("duckdb" %in% names(platforms)) + expect_true("sqlite" %in% names(platforms)) + + pgConfig <- platforms$postgresql$namespace$cg$tables$cohort_counts + expect_equal(pgConfig$partition_by, "RANGE (cohort_definition_id)") + expect_true(length(pgConfig$indexes) > 0) +}) + +test_that("yaml spec validation fails on invalid files", { + tmpYaml <- tempfile(fileext = ".yaml") + writeLines("namespace:\n test:\n tables:\n t1:\n columns:\n - name: col1", tmpYaml) + on.exit(unlink(tmpYaml)) + expect_error(loadResultsDataModelSpecifications(tmpYaml)) + + writeLines("version: '1.0'\nnamespace:\n test:\n tables:\n t1:\n columns:\n - type: int", tmpYaml) + expect_error(loadResultsDataModelSpecifications(tmpYaml)) +}) + +test_that("csv and yaml specs produce equivalent data for data model tables", { + csvSpec <- suppressWarnings( + loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv") + ) + yamlSpec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml") + + csvTables <- unique(csvSpec$tableName) + yamlTables <- unique(yamlSpec$tableName) + expect_setequal(csvTables, yamlTables) + + for (tbl in csvTables) { + csvCols <- csvSpec |> + dplyr::filter(.data$tableName == tbl) |> + dplyr::pull("columnName") |> + sort() + yamlCols <- yamlSpec |> + dplyr::filter(.data$tableName == tbl) |> + dplyr::pull("columnName") |> + sort() + expect_setequal(csvCols, yamlCols) + } +}) From 3b7130994fcea8695e4c6735fdcb03589aa4c928 Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 16:00:33 -0700 Subject: [PATCH 2/8] Updating documentation --- R/DataModel.R | 2 +- R/SchemaGenerator.R | 2 +- tests/testthat/test-SchemaGenerator.R | 11 +-- vignettes/ExampleProject.Rmd | 52 +++++++--- vignettes/UploadFunctionality.Rmd | 135 ++++++++++++++++++-------- vignettes/UsingQueryNamespaces.Rmd | 3 +- 6 files changed, 146 insertions(+), 59 deletions(-) diff --git a/R/DataModel.R b/R/DataModel.R index 9c16d09..bcfec99 100644 --- a/R/DataModel.R +++ b/R/DataModel.R @@ -958,7 +958,7 @@ loadResultsDataModelSpecifications <- function(filePath) { warning( "CSV-based results data model specifications are deprecated. ", "Use the namespaced YAML format instead. ", - "See the 'YAML Specification Format' vignette and the csvToYaml() function to migrate.", + "See vignette('UploadFunctionality') and the csvToYaml() function to migrate.", call. = FALSE ) spec <- readr::read_csv(file = filePath, col_types = readr::cols()) diff --git a/R/SchemaGenerator.R b/R/SchemaGenerator.R index b924877..217b09b 100644 --- a/R/SchemaGenerator.R +++ b/R/SchemaGenerator.R @@ -122,7 +122,7 @@ generateSqlSchema <- function(csvFilepath = NULL, warning( "CSV-based schema definitions are deprecated. ", "Use the namespaced YAML format instead. ", - "See the 'YAML Specification Format' vignette and the csvToYaml() function to migrate.", + "See vignette('UploadFunctionality') and the csvToYaml() function to migrate.", call. = FALSE ) schemaDefinition <- readr::read_csv(csvFilepath, show_col_types = FALSE) diff --git a/tests/testthat/test-SchemaGenerator.R b/tests/testthat/test-SchemaGenerator.R index e2815ad..8c7715c 100644 --- a/tests/testthat/test-SchemaGenerator.R +++ b/tests/testthat/test-SchemaGenerator.R @@ -8,21 +8,20 @@ test_that("Schema gen from file", { DatabaseConnector::disconnect(connection) }) - schema <- suppressWarnings( - generateSqlSchema(csvFilepath = "settings/testSchemaDef.csv", sqlOutputPath = tfile) - ) + schema <- generateSqlSchema(csvFilepath = "settings/testSchemaDef.yaml", sqlOutputPath = tfile) checkmate::expect_file_exists(tfile) - schemaDetails <- readr::read_csv("settings/testSchemaDef.csv", show_col_types = FALSE) - colnames(schemaDetails) <- SqlRender::snakeCaseToCamelCase(colnames(schemaDetails)) + schemaDetails <- loadResultsDataModelSpecifications("settings/testSchemaDef.yaml") checkmate::expect_string(schema) DatabaseConnector::renderTranslateExecuteSql(connection, schema, database_schema = "main") for (table in schemaDetails$tableName |> unique()) { + ns <- schemaDetails$namespace[schemaDetails$tableName == table][1] + dbTable <- paste0(ns, "_", table) res <- DatabaseConnector::renderTranslateQuerySql(connection, "SELECT * FROM @table_name", - table_name = table + table_name = dbTable ) } }) diff --git a/vignettes/ExampleProject.Rmd b/vignettes/ExampleProject.Rmd index 7e91b75..7188956 100644 --- a/vignettes/ExampleProject.Rmd +++ b/vignettes/ExampleProject.Rmd @@ -40,21 +40,49 @@ The aspects this package will cover are as follows: First we will create an R package called `SimpleFeatureExtractor` a toy example that pulls a set of aggregate features for specified cohorts from an OMOP CDM and exports the result set to a relational database for further analysis. -In this example we will export a single csv file that contains the following: +In this example we will export a single table that contains the following: -| table_name | column_name | data_type | is_required | primary_key | optional | empty_is_na | -|----------------------|-----------------------|-----------|-------------|-------------|----------|-------------| -| covariate_definition | covariate_id | int | Yes | Yes | No | No | -| covariate_definition | covariate_name | varchar | Yes | No | No | No | -| covariate_result | cohort_definition_id | int | Yes | Yes | No | No | -| covariate_result | covariate_id | bigint | Yes | Yes | No | No | -| covariate_result | covariate_mean | numeric | Yes | No | No | No | +| namespace | table_name | column_name | data_type | primary_key | optional | +|-----------|----------------------|-----------------------|-----------|-------------|----------| +| sfe | covariate_definition | covariate_id | int | Yes | No | +| sfe | covariate_definition | covariate_name | varchar | No | No | +| sfe | covariate_result | cohort_definition_id | int | Yes | No | +| sfe | covariate_result | covariate_id | bigint | Yes | No | +| sfe | covariate_result | covariate_mean | numeric | No | No | -Results exported from this package are covariate prevalances related to a cohorts and given a covariate_id, these are +Results exported from this package are covariate prevalences related to a cohorts and given a covariate_id, these are related to names in a second table. -This table should be saved to the `inst` folder of your R pacakge. -Preferably, this file should be called `resultsDataModelSpecification.csv` + +The recommended format is a namespaced YAML file saved to the `inst` folder of your R package. +This file should be called `resultsDataModelSpecification.yaml`: + +```yaml +version: "1.0" + +namespace: + sfe: + tables: + covariate_definition: + columns: + - name: covariate_id + type: int + primary_key: true + - name: covariate_name + type: varchar + covariate_result: + columns: + - name: cohort_definition_id + type: int + primary_key: true + - name: covariate_id + type: bigint + primary_key: true + - name: covariate_mean + type: numeric +``` + +CSV format is still supported but deprecated; use `csvToYaml()` to migrate existing CSV files. The package should create results csv files that correspond to these fields in terms of type and name. @@ -63,7 +91,7 @@ The package should create results csv files that correspond to these fields in t ## Creating a results database schema First we should load our specification ```{r,eval=FALSE} -specification <- ResultModelManager::loadResultsDataModelSpecifications("resultsDataModelSpecification") +specification <- ResultModelManager::loadResultsDataModelSpecifications("resultsDataModelSpecification.yaml") ``` We can then create our schema from this sql: diff --git a/vignettes/UploadFunctionality.Rmd b/vignettes/UploadFunctionality.Rmd index 8b84533..702a5cc 100644 --- a/vignettes/UploadFunctionality.Rmd +++ b/vignettes/UploadFunctionality.Rmd @@ -20,59 +20,118 @@ In the examples here, we assume the use of sqlite for simplicity. However, in principle any platform supported by the `DatabaseConnector` and `SqlRender` packages should work. # Creating a schema definition file -It is recommended that every analytics package that creates data output should contain a csv file. -This is a requirement for packages that use the OHDSI `Strategus` library. +Every analytics package that creates data output should contain a YAML or CSV specification that defines +its results data model. This package uses a **namespaced YAML format** by default; CSV support is deprecated. -Schema definitions should conform to the following column headers: +Schema definitions must include the following information per column: -``` -table_name, column_name, data_type, is_required, primary_key -``` +- **table name** (within a namespace) +- **column name** +- **data type** (e.g., `varchar`, `bigint`, `int`, `float`, `date`, `varchar(255)`) +- **primary key** indicator + +In addition, optional fields such as `optional`, `min_cell_count`, `empty_is_na`, and `description` +may be included. + +## YAML format (recommended) -In addition, other packages may make use of additional fields such as `optional` or `empty_is_na`. -These are not required for uploading to a schem. +A namespaced YAML specification looks like this: -An example csv file may look like this: +```yaml +version: "1.0" +namespace: + my_analysis: + tables: + table_1: + columns: + - name: database_id + type: varchar + primary_key: true + - name: analysis_id + type: bigint + primary_key: true + - name: analysis_name + type: varchar + - name: domain_id + type: varchar(20) + optional: true + - name: start_day + type: float + optional: true + - name: end_day + type: float + optional: true + - name: is_binary + type: varchar(1) + - name: missing_means_zero + type: varchar(1) + optional: true + table_2: + columns: + - name: database_id + type: varchar + primary_key: true + - name: analysis2_id + type: bigint + primary_key: true + - name: concept_id + type: int + - name: logic_description + type: varchar + optional: true + - name: valid_start_date + type: Date + - name: concept_name + type: varchar(255) + - name: p_10_value + type: float ``` + +The namespace name (e.g. `my_analysis`) becomes the table prefix in the database. +A table named `table_1` in namespace `my_analysis` will be created as `my_analysis_table_1`. + +Platform-specific features such as table partitioning and indexes can be configured +per DBMS platform. See `?loadResultsDataModelFromYaml` for details. + +## CSV format (deprecated) + +```{r eval=FALSE} # File inst/settings/resulsDataModelSpecifications.csv table_name,column_name,data_type,is_required,primary_key table_1,database_id,varchar,Yes,Yes table_1,analysis_id,bigint,Yes,Yes -table_1,analysis_name,varchar,Yes,No -table_1,domain_id,varchar(20),No,No -table_1,start_day,float,No,No -table_1,end_day,float,No,No -table_1,is_binary,varchar(1),Yes,No -table_1,missing_means_zero,varchar(1),No,No -table_2,database_id,varchar,Yes,Yes -table_2,analysis2_id,bigint,Yes,Yes -table_2,concept_id,int,Yes,No -table_2,logic_description,varchar,No,No -table_2,valid_start_date,Date,Yes,No -table_2,concept_name,varchar(255),Yes,No -table_2,p_10_value,float,Yes,No -table_3,database_id,varchar,Yes,Yes -table_3,analysis3_id,bigint,Yes,Yes -table_3,concept_id,int,Yes,No -table_3,logic_description,varchar,No,No -table_3,valid_start_date,Date,Yes,No -table_3,concept_name,varchar(255),Yes,No -table_3,p_10_value,float,Yes,No +... ``` -Note the use of sql server data types in the `data_type` field as well as `yes` and `no` in the binary field `is_required` -and `primary_key`. +CSV-based specifications are deprecated. Use `csvToYaml()` to migrate existing CSV files +to the YAML format. + +## Loading specifications + +To load a specification file: + +```{r eval = FALSE} +# YAML (recommended) +spec <- ResultModelManager::loadResultsDataModelSpecifications("path/to/spec.yaml") + +# The YAML specification is loaded via +result <- ResultModelManager::loadResultsDataModelFromYaml("path/to/spec.yaml") +spec <- result$specification # data.frame +platforms <- result$platforms # platform-specific config + +# CSV (deprecated) +spec <- ResultModelManager::loadResultsDataModelSpecifications("path/to/spec.csv") +``` -We should also define a function for loading this file that converts the column headers to camel case format. +For loading inside an R package via `system.file`: ```{r eval = FALSE} -#' Get Results Data Model Specifcations +#' Get Results Data Model Specifications getResultsDataModelSpec <- function() { - # For loading inside an R package - specPath <- system.file("settings", "resulsDataModelSpecifications.csv", package = utils::packageName()) - spec <- readr::read_csv(specPath, show_col_types = FALSE) - colnames(spec) <- SqlRender::snakeCaseToCamelCase(colnames(spec)) + specPath <- system.file("settings", "resultsDataModelSpecification.yaml", + package = utils::packageName()) + spec <- ResultModelManager::loadResultsDataModelSpecifications(specPath) return(spec) } ``` @@ -93,7 +152,7 @@ parameter `table_prefix`. # Uploading results This section shows how to upload results conforming to the above specified schema. It is assumed that a zip file has been created with csv files corresponding to the table names provided in the -`resulsDataModelSpecifications.csv` file. +`resultsDataModelSpecification` file. ```{r eval=FALSE} ResultModelManager::unzipResults(zipFile = "MyResultsZip.zip", resultsFolder = "extraction_folder") diff --git a/vignettes/UsingQueryNamespaces.Rmd b/vignettes/UsingQueryNamespaces.Rmd index bcd144c..56a065e 100644 --- a/vignettes/UsingQueryNamespaces.Rmd +++ b/vignettes/UsingQueryNamespaces.Rmd @@ -38,7 +38,8 @@ tableSpecification <- data.frame( dataType = c("bigint", "varchar", "varchar", "varchar") ) ``` -Note, that generally we would save these tables to a csv file that can be loaded. +Note, that generally we would save these tables to a yaml file that can be loaded +(see the "Upload Functionality" vignette for the YAML schema format). We then load a `QueryNamespace` instance with this table: From d602b7b3572780db2dd459465c6b923b41bf4426 Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 16:02:41 -0700 Subject: [PATCH 3/8] Updated tests to use new yaml format --- tests/testthat/test-QueryNamespace.R | 32 ++++++++++++------------- tests/testthat/test-Utils.R | 4 +--- tests/testthat/test-YamlNamespace.R | 10 ++++---- tests/testthat/test-YamlSpecification.R | 21 ++++++++-------- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/tests/testthat/test-QueryNamespace.R b/tests/testthat/test-QueryNamespace.R index e94794a..714278f 100644 --- a/tests/testthat/test-QueryNamespace.R +++ b/tests/testthat/test-QueryNamespace.R @@ -78,9 +78,7 @@ test_that("create helper function works", { qns <- createQueryNamespace( connectionDetails = connectionDetails, usePooledConnection = FALSE, - tableSpecification = suppressWarnings( - loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv") - ), + tableSpecification = loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml"), resultModelSpecificationPath = NULL, tablePrefix = "", snakeCaseToCamelCase = TRUE, @@ -90,27 +88,27 @@ test_that("create helper function works", { vars <- qns$getVars() expect_true("databaseSchema" %in% names(vars)) - qns <- suppressWarnings(createQueryNamespace( + qns <- createQueryNamespace( connectionDetails = connectionDetails, usePooledConnection = TRUE, tableSpecification = NULL, resultModelSpecificationPath = c( - "settings/resultsDataModelSpecification.csv", - "settings/testSchemaDef.csv" + "settings/resultsDataModelSpecification.yaml", + "settings/testSchemaDef.yaml" ), tablePrefix = "", snakeCaseToCamelCase = TRUE, databaseSchema = "main" - )) + ) vars <- qns$getVars() - expect_true("cohort_counts" %in% names(vars)) - expect_true("cohort_definition" %in% names(vars)) - expect_true("cdm_source_info" %in% names(vars)) - expect_true("cosine_similarity" %in% names(vars)) - expect_true("covariate_definition" %in% names(vars)) - expect_true("covariate_mean" %in% names(vars)) - expect_true("test_table_1" %in% names(vars)) + expect_true("cg_cohort_counts" %in% names(vars)) + expect_true("cg_cohort_definition" %in% names(vars)) + expect_true("cg_cdm_source_info" %in% names(vars)) + expect_true("cg_cosine_similarity" %in% names(vars)) + expect_true("cg_covariate_definition" %in% names(vars)) + expect_true("cg_covariate_mean" %in% names(vars)) + expect_true("test_test_table_1" %in% names(vars)) expect_error( createQueryNamespace( @@ -125,15 +123,15 @@ test_that("create helper function works", { ) expect_error( - suppressWarnings(createQueryNamespace( + createQueryNamespace( connectionDetails = NULL, usePooledConnection = FALSE, tableSpecification = NULL, - resultModelSpecificationPath = c("settings/resultsDataModelSpecification.csv"), + resultModelSpecificationPath = c("settings/resultsDataModelSpecification.yaml"), tablePrefix = "", snakeCaseToCamelCase = TRUE, databaseSchema = "main" - )) + ) ) }) diff --git a/tests/testthat/test-Utils.R b/tests/testthat/test-Utils.R index 3d2c63f..e91deea 100644 --- a/tests/testthat/test-Utils.R +++ b/tests/testthat/test-Utils.R @@ -1,7 +1,5 @@ test_that("Grant permissions", { - tableSpecification <- suppressWarnings( - loadResultsDataModelSpecifications("settings/testSchemaDef.csv") - ) + tableSpecification <- loadResultsDataModelSpecifications("settings/testSchemaDef.yaml") expect_error( grantTablePermissions( diff --git a/tests/testthat/test-YamlNamespace.R b/tests/testthat/test-YamlNamespace.R index 1802692..4922687 100644 --- a/tests/testthat/test-YamlNamespace.R +++ b/tests/testthat/test-YamlNamespace.R @@ -97,23 +97,23 @@ test_that("createQueryNamespace works with yaml spec files", { expect_equal(vars[["cg_cohort_definition"]], "test_cg_cohort_definition") }) -test_that("createQueryNamespace merges csv and yaml specs", { +test_that("createQueryNamespace merges two yaml specs", { skip_on_cran() - qns <- suppressWarnings(createQueryNamespace( + qns <- createQueryNamespace( connectionDetails = connectionDetails, usePooledConnection = FALSE, resultModelSpecificationPath = c( - "settings/resultsDataModelSpecification.csv", + "settings/resultsDataModelSpecification.yaml", "settings/testSchemaDef.yaml" ), tablePrefix = "", snakeCaseToCamelCase = TRUE, databaseSchema = "main" - )) + ) vars <- qns$getVars() - expect_true("test_table_1" %in% names(vars)) + expect_true("test_test_table_1" %in% names(vars)) expect_true("cg_cohort_definition" %in% names(vars)) expect_true("cg_cohort_counts" %in% names(vars)) }) diff --git a/tests/testthat/test-YamlSpecification.R b/tests/testthat/test-YamlSpecification.R index 44fa2c8..b7738ba 100644 --- a/tests/testthat/test-YamlSpecification.R +++ b/tests/testthat/test-YamlSpecification.R @@ -53,18 +53,17 @@ test_that("yaml spec validation fails on invalid files", { expect_error(loadResultsDataModelSpecifications(tmpYaml)) }) -test_that("csv and yaml specs produce equivalent data for data model tables", { - csvSpec <- suppressWarnings( - loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.csv") - ) - yamlSpec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml") +test_that("loadResultsDataModelSpecifications and loadResultsDataModelFromYaml produce same data", { + directSpec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml") + fromYaml <- loadResultsDataModelFromYaml("settings/resultsDataModelSpecification.yaml") + yamlSpec <- fromYaml$specification - csvTables <- unique(csvSpec$tableName) - yamlTables <- unique(yamlSpec$tableName) - expect_setequal(csvTables, yamlTables) + directTables <- sort(unique(directSpec$tableName)) + yamlTables <- sort(unique(yamlSpec$tableName)) + expect_setequal(directTables, yamlTables) - for (tbl in csvTables) { - csvCols <- csvSpec |> + for (tbl in directTables) { + directCols <- directSpec |> dplyr::filter(.data$tableName == tbl) |> dplyr::pull("columnName") |> sort() @@ -72,6 +71,6 @@ test_that("csv and yaml specs produce equivalent data for data model tables", { dplyr::filter(.data$tableName == tbl) |> dplyr::pull("columnName") |> sort() - expect_setequal(csvCols, yamlCols) + expect_setequal(directCols, yamlCols) } }) From 698b5d441e4d6a26bc656317e1c1800a76630c0f Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 16:39:10 -0700 Subject: [PATCH 4/8] fixing broken tests --- R/QueryNamespace.R | 28 +++++++++++++++++++++------- R/Utils.R | 11 ++++++++++- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R index 3a1290f..0050e82 100644 --- a/R/QueryNamespace.R +++ b/R/QueryNamespace.R @@ -207,17 +207,31 @@ QueryNamespace <- R6::R6Class( hasNamespace <- "namespace" %in% colnames(tableSpecification) - tableEntries <- tableSpecification |> - dplyr::select(dplyr::any_of(c("namespace", "tableName"))) |> - dplyr::distinct() + if (hasNamespace) { + tableEntries <- tableSpecification |> + dplyr::select("namespace", "tableName") |> + dplyr::distinct() + } else { + tableEntries <- tableSpecification |> + dplyr::select("tableName") |> + dplyr::distinct() + } for (i in seq_len(nrow(tableEntries))) { - ns <- tableEntries$namespace[i] tableName <- tableEntries$tableName[i] - if (hasNamespace && !is.na(ns)) { - registryKey <- paste0(ns, "_", tableName) - replacementVar <- paste0(tablePrefix, ns, "_", tableName) + if (hasNamespace) { + ns <- tableEntries$namespace[i] + if (!is.na(ns)) { + registryKey <- paste0(ns, "_", tableName) + replacementVar <- paste0(tablePrefix, ns, "_", tableName) + } else if (useTablePrefix) { + registryKey <- tableName + replacementVar <- paste0(tablePrefix, tableName) + } else { + registryKey <- tableName + replacementVar <- tableName + } } else if (useTablePrefix) { registryKey <- tableName replacementVar <- paste0(tablePrefix, tableName) diff --git a/R/Utils.R b/R/Utils.R index 47bba71..45bf458 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -58,12 +58,21 @@ grantTablePermissions <- function(connectionDetails = NULL, warning("Untested grant function for this database platform") } + hasNamespace <- "namespace" %in% colnames(tableSpecification) + sql <- c() for (table in tableSpecification$tableName |> unique()) { + tableVar <- table + if (hasNamespace) { + ns <- tableSpecification$namespace[tableSpecification$tableName == table][1] + if (!is.na(ns)) { + tableVar <- paste0(ns, "_", table) + } + } sql <- c( sql, SqlRender::render("GRANT @permissions ON @database_schema.@table_prefix@table TO @user;", - table = table, + table = tableVar, table_prefix = tablePrefix ) ) From cf9ab1cd09d18358fb25a026d6909a897895e633 Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 16:53:33 -0700 Subject: [PATCH 5/8] Updated to allow nullable tables and have more strict data structures. Clarified documentation in line with changes --- R/DataModel.R | 72 +++++++++++-- R/QueryNamespace.R | 16 ++- R/SchemaGenerator.R | 12 ++- README.md | 48 ++++++++- .../resultsDataModelSpecification.yaml | 13 +-- tests/testthat/settings/testSchemaDef.yaml | 27 +++-- tests/testthat/test-YamlSpecification.R | 43 +++++++- vignettes/ExampleProject.Rmd | 37 +++++-- vignettes/PackageDesign.Rmd | 2 +- vignettes/UploadFunctionality.Rmd | 101 ++++++++++++++---- vignettes/UsingQueryNamespaces.Rmd | 34 ++++-- 11 files changed, 333 insertions(+), 72 deletions(-) diff --git a/R/DataModel.R b/R/DataModel.R index bcfec99..a428e6a 100644 --- a/R/DataModel.R +++ b/R/DataModel.R @@ -861,11 +861,15 @@ checkSpecificationColumns <- function(columnNames) { .yamlColDefaults <- list( primaryKey = "No", + nullable = "No", isRequired = "Yes", optional = "No", minCellCount = "No", emptyIsNa = "Yes", - description = NA_character_ + references = NA_character_, + description = NA_character_, + tableDescription = NA_character_, + namespacePrefix = NA_character_ ) .yamlSpecToDataFrame <- function(yamlContent) { @@ -877,6 +881,13 @@ checkSpecificationColumns <- function(columnNames) { for (nsName in names(yamlContent$namespace)) { nsDef <- yamlContent$namespace[[nsName]] + + nsPrefix <- nsDef$prefix + if (is.null(nsPrefix)) { + nsPrefix <- paste0(nsName, "_") + } + nsDescription <- nsDef$description %||% NA_character_ + if (!"tables" %in% names(nsDef)) { stop(sprintf("Namespace '%s' must contain a 'tables' key", nsName)) } @@ -887,6 +898,8 @@ checkSpecificationColumns <- function(columnNames) { stop(sprintf("Table '%s' in namespace '%s' must contain a 'columns' key", tableName, nsName)) } + tblDescription <- tableDef$description %||% .yamlColDefaults$tableDescription + for (col in tableDef$columns) { if (is.null(col$name)) { stop(sprintf("Column in table '%s' (namespace '%s') must have a 'name'", tableName, nsName)) @@ -895,17 +908,32 @@ checkSpecificationColumns <- function(columnNames) { stop(sprintf("Column '%s' in table '%s' (namespace '%s') must have a 'type'", col$name, tableName, nsName)) } + colNullable <- FALSE + if (!is.null(col$nullable)) { + colNullable <- isTRUE(col$nullable) + } else if (isTRUE(col$optional)) { + colNullable <- TRUE + } else if (isFALSE(col$required)) { + colNullable <- TRUE + } + + colReferences <- col$references %||% .yamlColDefaults$references + row <- list( namespace = nsName, + namespacePrefix = nsPrefix, tableName = tableName, + tableDescription = tblDescription, columnName = col$name, dataType = col$type, primaryKey = ifelse(isTRUE(col$primary_key), "Yes", .yamlColDefaults$primaryKey), - isRequired = ifelse(isFALSE(col$required), "No", .yamlColDefaults$isRequired), - optional = ifelse(isTRUE(col$optional), "Yes", .yamlColDefaults$optional), + nullable = ifelse(colNullable, "Yes", "No"), + isRequired = ifelse(colNullable, "No", "Yes"), + optional = ifelse(colNullable, "Yes", "No"), minCellCount = ifelse(isTRUE(col$min_cell_count), "Yes", .yamlColDefaults$minCellCount), emptyIsNa = ifelse(isFALSE(col$empty_is_na), "No", .yamlColDefaults$emptyIsNa), - description = if (is.null(col$description)) .yamlColDefaults$description else col$description + references = colReferences, + description = col$description %||% .yamlColDefaults$description ) rows[[length(rows) + 1]] <- row } @@ -922,9 +950,21 @@ checkSpecificationColumns <- function(columnNames) { #' Load a YAML results data model specification file. Returns a list containing #' the flattened specification data frame and optional platform-specific configuration. #' +#' Each HADES package should ship its own \code{resultsDataModelSpecification.yaml} in +#' its \code{inst/settings/} directory. A package's YAML defines only the tables +#' produced by that package. +#' +#' Combining multiple YAML specifications (e.g. from different packages) into a +#' unified HADES results schema is the responsibility of a higher-level +#' orchestrator such as Strategus or a future \code{OhdsiResultsManager} package. +#' This package provides per-spec loading only, to avoid circular dependencies +#' between packages that share no direct relationship at install time. +#' #' @param filePath Path to a valid YAML file #' @return A list with elements: -#' \item{specification}{A tibble data frame with columns: namespace, tableName, columnName, dataType, primaryKey, isRequired, optional, minCellCount, emptyIsNa, description} +#' \item{specification}{A tibble data frame with columns: namespace, namespacePrefix, +#' tableName, tableDescription, columnName, dataType, primaryKey, nullable, +#' isRequired, optional, minCellCount, emptyIsNa, references, description} #' \item{platforms}{Platform-specific configuration (postgresql, sql_server, duckdb, sqlite) or NULL} #' #' @export @@ -1008,26 +1048,35 @@ csvToYaml <- function(csvFilepath, pk <- tolower(tblSpec$primaryKey[i]) if (pk == "yes") { col$primary_key <- TRUE + } else { + col$primary_key <- FALSE } + colNullable <- FALSE if ("optional" %in% colnames(tblSpec)) { opt <- tolower(tblSpec$optional[i]) if (opt == "yes") { - col$optional <- TRUE + colNullable <- TRUE } } - if ("isRequired" %in% colnames(tblSpec)) { req <- tolower(tblSpec$isRequired[i]) if (req == "no") { - col$required <- FALSE + colNullable <- TRUE } } + if (colNullable) { + col$nullable <- TRUE + } else { + col$nullable <- FALSE + } if ("minCellCount" %in% colnames(tblSpec)) { mcc <- tolower(tblSpec$minCellCount[i]) if (mcc == "yes") { col$min_cell_count <- TRUE + } else { + col$min_cell_count <- FALSE } } @@ -1035,6 +1084,8 @@ csvToYaml <- function(csvFilepath, ein <- tolower(tblSpec$emptyIsNa[i]) if (ein == "no") { col$empty_is_na <- FALSE + } else { + col$empty_is_na <- TRUE } } @@ -1048,11 +1099,14 @@ csvToYaml <- function(csvFilepath, tables[[tbl]] <- list(columns = columns) } + nsConfig <- list(tables = tables) + nsConfig$prefix <- paste0(namespace, "_") + yamlContent <- list( version = "1.0", namespace = list() ) - yamlContent$namespace[[namespace]] <- list(tables = tables) + yamlContent$namespace[[namespace]] <- nsConfig yamlStr <- yaml::as.yaml(yamlContent, indent.mapping.sequence = TRUE) diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R index 0050e82..949ca35 100644 --- a/R/QueryNamespace.R +++ b/R/QueryNamespace.R @@ -206,10 +206,11 @@ QueryNamespace <- R6::R6Class( assertSpecificationColumns(colnames(tableSpecification)) hasNamespace <- "namespace" %in% colnames(tableSpecification) + hasNsPrefix <- "namespacePrefix" %in% colnames(tableSpecification) if (hasNamespace) { tableEntries <- tableSpecification |> - dplyr::select("namespace", "tableName") |> + dplyr::select(dplyr::any_of(c("namespace", "namespacePrefix", "tableName"))) |> dplyr::distinct() } else { tableEntries <- tableSpecification |> @@ -223,8 +224,17 @@ QueryNamespace <- R6::R6Class( if (hasNamespace) { ns <- tableEntries$namespace[i] if (!is.na(ns)) { - registryKey <- paste0(ns, "_", tableName) - replacementVar <- paste0(tablePrefix, ns, "_", tableName) + nsPrefix <- NULL + if (hasNsPrefix) { + nsPrefix <- tableEntries$namespacePrefix[i] + } + if (!is.null(nsPrefix) && !is.na(nsPrefix)) { + registryKey <- paste0(ns, "_", tableName) + replacementVar <- paste0(tablePrefix, nsPrefix, tableName) + } else { + registryKey <- paste0(ns, "_", tableName) + replacementVar <- paste0(tablePrefix, ns, "_", tableName) + } } else if (useTablePrefix) { registryKey <- tableName replacementVar <- paste0(tablePrefix, tableName) diff --git a/R/SchemaGenerator.R b/R/SchemaGenerator.R index 217b09b..20085b7 100644 --- a/R/SchemaGenerator.R +++ b/R/SchemaGenerator.R @@ -132,6 +132,7 @@ generateSqlSchema <- function(csvFilepath = NULL, assertSpecificationColumns(colnames(schemaDefinition)) hasNamespace <- "namespace" %in% colnames(schemaDefinition) + hasNamespacePrefix <- "namespacePrefix" %in% colnames(schemaDefinition) tableSqlStr <- " CREATE TABLE @database_schema.@table_prefix@table_name ( @@ -157,7 +158,16 @@ CREATE TABLE @database_schema.@table_prefix@table_name ( if (hasNamespace) { ns <- tableColumns$namespace[1] if (!is.na(ns)) { - varName <- paste0(ns, "_", table) + if (hasNamespacePrefix) { + nsPrefix <- tableColumns$namespacePrefix[1] + if (!is.na(nsPrefix)) { + varName <- paste0(nsPrefix, table) + } else { + varName <- paste0(ns, "_", table) + } + } else { + varName <- paste0(ns, "_", table) + } } } diff --git a/README.md b/README.md index 355dc8e..7a27fe8 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,52 @@ ResultModelManager (RMM) [HADES](https://ohdsi.github.io/Hades/). Introduction ============ -RMM is a database data model management utilities for R packages in the Observational Health Data Sciences and Informatics programme (OHDSI). RMM provides utility functions to -allow package maintainers to migrate existing SQL database models, export and import results in consistent patterns. +RMM is a database data model management utility for R packages in the Observational Health Data Sciences and Informatics programme (OHDSI). RMM provides utility functions to +allow package maintainers to define results data models in YAML, migrate existing SQL database models, export and import results in consistent patterns. + +YAML-based Data Model Specifications +===================================== + +Package maintainers define their results data model in a namespaced YAML file +shipped within their package (`inst/settings/resultsDataModelSpecification.yaml`): + +```yaml +version: "1.0" + +namespace: + mypackage: + prefix: mp_ + tables: + my_table: + columns: + - name: database_id + type: varchar + primary_key: true + - name: result_id + type: bigint + primary_key: true + - name: result_value + type: float + nullable: true +``` + +Each HADES package owns its own specification — there is no central results model +repository. This avoids circular dependencies between packages. Multiple +specifications are combined by higher-level orchestrators such as Strategus. + +Platform-specific features (partitioning, indexes) are configured in the same file: + +```yaml +platforms: + postgresql: + namespace: + mypackage: + tables: + my_table: + partition_by: RANGE (result_id) +``` + +CSV-based specifications are deprecated; use `csvToYaml()` to migrate. System Requirements diff --git a/tests/testthat/settings/resultsDataModelSpecification.yaml b/tests/testthat/settings/resultsDataModelSpecification.yaml index a4ed6e1..37edff4 100644 --- a/tests/testthat/settings/resultsDataModelSpecification.yaml +++ b/tests/testthat/settings/resultsDataModelSpecification.yaml @@ -2,6 +2,7 @@ version: "1.0" namespace: test: + prefix: test_ description: "Test namespace for basic upload/validation" tables: test_table_1: @@ -16,18 +17,18 @@ namespace: type: varchar - name: domain_id type: varchar(20) - optional: true + nullable: true - name: start_day type: float - optional: true + nullable: true - name: end_day type: float - optional: true + nullable: true - name: is_binary type: varchar(1) - name: missing_means_zero type: varchar(1) - optional: true + nullable: true test_table_2: columns: - name: database_id @@ -40,7 +41,7 @@ namespace: type: int - name: logic_description type: varchar - optional: true + nullable: true - name: valid_start_date type: Date - name: concept_name @@ -59,7 +60,7 @@ namespace: type: int - name: logic_description type: varchar - optional: true + nullable: true - name: valid_start_date type: Date - name: concept_name diff --git a/tests/testthat/settings/testSchemaDef.yaml b/tests/testthat/settings/testSchemaDef.yaml index d9807e4..e475971 100644 --- a/tests/testthat/settings/testSchemaDef.yaml +++ b/tests/testthat/settings/testSchemaDef.yaml @@ -2,13 +2,16 @@ version: "1.0" namespace: cg: + prefix: cg_ description: "Cohort Generator namespace" tables: cohort_definition: + description: "Stores cohort definitions" columns: - name: cohort_definition_id type: bigint primary_key: true + description: "The unique identifier for the cohort definition" - name: concept_id type: bigint - name: cohort_name @@ -20,46 +23,52 @@ namespace: - name: database_id type: bigint primary_key: true + description: "The database identifier" cdm_source_info: + description: "CDM source information" columns: - name: database_id type: bigint primary_key: true + references: cg_cohort_definition.database_id - name: cdm_source_abbreviation type: varchar - optional: true + nullable: true - name: cdm_holder type: varchar - optional: true + nullable: true - name: source_description type: varchar - optional: true + nullable: true - name: source_documentation_reference type: varchar - optional: true + nullable: true - name: cdm_etl_reference type: varchar - optional: true + nullable: true - name: source_release_date type: date - optional: true + nullable: true - name: cdm_release_date type: date - optional: true + nullable: true - name: cdm_version type: varchar - optional: true + nullable: true - name: vocabulary_version type: varchar - optional: true + nullable: true cohort_counts: + description: "Counts of subjects per cohort" columns: - name: num_persons type: bigint min_cell_count: true + description: "Number of persons (min cell count enforced)" - name: cohort_definition_id type: bigint primary_key: true + references: cg_cohort_definition.cohort_definition_id - name: database_id type: bigint primary_key: true diff --git a/tests/testthat/test-YamlSpecification.R b/tests/testthat/test-YamlSpecification.R index b7738ba..8e29cd1 100644 --- a/tests/testthat/test-YamlSpecification.R +++ b/tests/testthat/test-YamlSpecification.R @@ -1,15 +1,34 @@ test_that("loadResultsDataModelSpecifications supports yaml files", { spec <- loadResultsDataModelSpecifications("settings/resultsDataModelSpecification.yaml") expect_true("namespace" %in% colnames(spec)) + expect_true("namespacePrefix" %in% colnames(spec)) expect_true("tableName" %in% colnames(spec)) + expect_true("tableDescription" %in% colnames(spec)) expect_true("columnName" %in% colnames(spec)) expect_true("dataType" %in% colnames(spec)) expect_true("primaryKey" %in% colnames(spec)) + expect_true("nullable" %in% colnames(spec)) + expect_true("isRequired" %in% colnames(spec)) + expect_true("optional" %in% colnames(spec)) + expect_true("references" %in% colnames(spec)) expect_equal(unique(spec$namespace), "test") + expect_equal(unique(spec$namespacePrefix), "test_") expect_true("test_table_1" %in% spec$tableName) expect_true("test_table_2" %in% spec$tableName) expect_true("test_table_3" %in% spec$tableName) + + nbRows <- spec |> + dplyr::filter(.data$columnName == "domain_id" & .data$tableName == "test_table_1") + expect_equal(nbRows$nullable, "Yes") + expect_equal(nbRows$isRequired, "No") + expect_equal(nbRows$optional, "Yes") + + reqRows <- spec |> + dplyr::filter(.data$columnName == "database_id" & .data$tableName == "test_table_1") + expect_equal(reqRows$nullable, "No") + expect_equal(reqRows$isRequired, "Yes") + expect_equal(reqRows$optional, "No") }) test_that("loadResultsDataModelSpecifications still works with csv files and warns deprecation", { @@ -23,14 +42,36 @@ test_that("loadResultsDataModelSpecifications still works with csv files and war expect_equal(unique(spec$tableName), c("test_table_1", "test_table_2", "test_table_3")) }) -test_that("loadResultsDataModelFromYaml returns platform config", { +test_that("loadResultsDataModelFromYaml returns platform config and rich metadata", { result <- loadResultsDataModelFromYaml("settings/testSchemaDef.yaml") expect_true("specification" %in% names(result)) expect_true("platforms" %in% names(result)) spec <- result$specification expect_true("namespace" %in% colnames(spec)) + expect_true("namespacePrefix" %in% colnames(spec)) + expect_true("tableDescription" %in% colnames(spec)) + expect_true("nullable" %in% colnames(spec)) + expect_true("references" %in% colnames(spec)) expect_equal(unique(spec$namespace), "cg") + expect_equal(unique(spec$namespacePrefix), "cg_") + + tblDesc <- spec$tableDescription[spec$tableName == "cohort_definition"][1] + expect_equal(tblDesc, "Stores cohort definitions") + + refRows <- spec |> + dplyr::filter(.data$columnName == "database_id" & .data$tableName == "cdm_source_info") + expect_equal(refRows$references, "cg_cohort_definition.database_id") + + cdIdRef <- spec |> + dplyr::filter(.data$columnName == "cohort_definition_id" & .data$tableName == "cohort_counts") + expect_equal(cdIdRef$references, "cg_cohort_definition.cohort_definition_id") + + nullableRows <- spec |> + dplyr::filter(.data$columnName == "cdm_source_abbreviation") + expect_equal(nullableRows$nullable, "Yes") + expect_equal(nullableRows$isRequired, "No") + expect_equal(nullableRows$optional, "Yes") platforms <- result$platforms expect_true("postgresql" %in% names(platforms)) diff --git a/vignettes/ExampleProject.Rmd b/vignettes/ExampleProject.Rmd index 7188956..bb6c140 100644 --- a/vignettes/ExampleProject.Rmd +++ b/vignettes/ExampleProject.Rmd @@ -16,16 +16,20 @@ vignette: > ```{r,echo=FALSE} specification <- data.frame( + namespace = "sfe", + namespacePrefix = "sfe_", tableName = c("covariate_definition", "covariate_definition", "covariate_result", "covariate_result", "covariate_result"), columnName = c("covariate_id", "covariate_name", "cohort_definition_id", "covariate_id", "covariate_mean"), dataType = c("int", "varchar", "int", "bigint", "numeric"), primaryKey = c("Yes", "No", "Yes", "Yes", "No"), + nullable = c("No", "No", "No", "No", "No"), + isRequired = c("Yes", "Yes", "Yes", "Yes", "Yes"), optional = c("No", "No", "No", "No", "No") ) ``` # Introduction -This guide intends to server as an example of using RMM to build and maintain a package that produces results in an +This guide intends to serve as an example of using RMM to build and maintain a package that produces results in an end to end manner. The aspects this package will cover are as follows: @@ -40,9 +44,9 @@ The aspects this package will cover are as follows: First we will create an R package called `SimpleFeatureExtractor` a toy example that pulls a set of aggregate features for specified cohorts from an OMOP CDM and exports the result set to a relational database for further analysis. -In this example we will export a single table that contains the following: +In this example we will export two tables: -| namespace | table_name | column_name | data_type | primary_key | optional | +| namespace | table_name | column_name | data_type | primary_key | nullable | |-----------|----------------------|-----------------------|-----------|-------------|----------| | sfe | covariate_definition | covariate_id | int | Yes | No | | sfe | covariate_definition | covariate_name | varchar | No | No | @@ -50,11 +54,10 @@ In this example we will export a single table that contains the following: | sfe | covariate_result | covariate_id | bigint | Yes | No | | sfe | covariate_result | covariate_mean | numeric | No | No | - -Results exported from this package are covariate prevalences related to a cohorts and given a covariate_id, these are +Results exported from this package are covariate prevalences related to cohorts, given a covariate_id, these are related to names in a second table. -The recommended format is a namespaced YAML file saved to the `inst` folder of your R package. +The recommended format is a namespaced YAML file saved to the `inst/settings/` folder of your R package. This file should be called `resultsDataModelSpecification.yaml`: ```yaml @@ -62,26 +65,36 @@ version: "1.0" namespace: sfe: + prefix: sfe_ + description: "Simple Feature Extractor results" tables: covariate_definition: columns: - name: covariate_id type: int primary_key: true + description: "The unique covariate identifier" - name: covariate_name type: varchar + description: "Human-readable covariate name" covariate_result: columns: - name: cohort_definition_id type: int primary_key: true + references: sfe_covariate_definition.covariate_id - name: covariate_id type: bigint primary_key: true + references: sfe_covariate_definition.covariate_id - name: covariate_mean type: numeric ``` +Each HADES package owns its own YAML specification. There is no central results model repository — +packages ship their schema definitions independently. Combining multiple specs into a unified +schema is handled by higher-level orchestrators (e.g. Strategus). + CSV format is still supported but deprecated; use `csvToYaml()` to migrate existing CSV files. The package should create results csv files that correspond to these fields in terms of type and name. @@ -91,7 +104,9 @@ The package should create results csv files that correspond to these fields in t ## Creating a results database schema First we should load our specification ```{r,eval=FALSE} -specification <- ResultModelManager::loadResultsDataModelSpecifications("resultsDataModelSpecification.yaml") +specification <- ResultModelManager::loadResultsDataModelSpecifications( + system.file("settings", "resultsDataModelSpecification.yaml", package = "SimpleFeatureExtractor") +) ``` We can then create our schema from this sql: @@ -117,7 +132,7 @@ qns <- ResultModelManager::createQueryNamespace( database_schema = "main" ) -# note - the table prefix and schema parameters are not neeeded when we do this +# The schema DDL references namespaced table names (e.g. @sfe_covariate_definition) qns$executeSql(sql) ``` Alternatively, we can just use `DatabaseConnector` functions directly. @@ -152,14 +167,14 @@ ResultModelManager::uploadResults(connectionDetails, ) ``` -With the results uploaded we can now write queries inside the namespace: +With the results uploaded we can now write queries inside the namespace using the namespaced table references: ```{r} -qns$queryDb("SELECT * FROM @database_schema.@covariate_definition") +qns$queryDb("SELECT * FROM @database_schema.@sfe_covariate_definition") ``` ```{r} -qns$queryDb("SELECT * FROM @database_schema.@covariate_result WHERE cohort_definition_id = @cohort_id", +qns$queryDb("SELECT * FROM @database_schema.@sfe_covariate_result WHERE cohort_definition_id = @cohort_id", cohort_id = 5 ) ``` diff --git a/vignettes/PackageDesign.Rmd b/vignettes/PackageDesign.Rmd index a71b819..06b29e4 100644 --- a/vignettes/PackageDesign.Rmd +++ b/vignettes/PackageDesign.Rmd @@ -207,7 +207,7 @@ It is desirable, but not a hard requirement for the initial package version, to To provide the ability to add a new migration to an existing project ### Create DDL design -Function to create a ddl csv that conforms to common standards +Function to create a YAML specification that conforms to common standards. ### Create new DDL In principle this could be added for new projects that will generate results. diff --git a/vignettes/UploadFunctionality.Rmd b/vignettes/UploadFunctionality.Rmd index 702a5cc..1895220 100644 --- a/vignettes/UploadFunctionality.Rmd +++ b/vignettes/UploadFunctionality.Rmd @@ -20,34 +20,52 @@ In the examples here, we assume the use of sqlite for simplicity. However, in principle any platform supported by the `DatabaseConnector` and `SqlRender` packages should work. # Creating a schema definition file -Every analytics package that creates data output should contain a YAML or CSV specification that defines -its results data model. This package uses a **namespaced YAML format** by default; CSV support is deprecated. +Every analytics package that creates data output should contain a **YAML specification** that defines +its results data model. The format is a namespaced YAML schema; CSV support is deprecated +(use `csvToYaml()` to migrate existing CSV files). + +## Schema ownership philosophy + +Each HADES package ships its own `resultsDataModelSpecification.yaml` in its `inst/settings/` directory. +A package's YAML defines **only the tables produced by that package**. + +This avoids the circular dependency problem of a central results model repository: +packages do not need to know about each other's schemas. Combining specifications from multiple +packages into a unified HADES results schema is the responsibility of a higher-level orchestrator +such as Strategus. + +## YAML format (recommended) Schema definitions must include the following information per column: -- **table name** (within a namespace) -- **column name** -- **data type** (e.g., `varchar`, `bigint`, `int`, `float`, `date`, `varchar(255)`) +- **column name** and **data type** (e.g., `varchar`, `bigint`, `int`, `float`, `date`, `varchar(255)`) - **primary key** indicator +- **nullable** flag (whether the column may contain NULL values) -In addition, optional fields such as `optional`, `min_cell_count`, `empty_is_na`, and `description` -may be included. +Additional optional fields: -## YAML format (recommended) +- `min_cell_count` — marks columns requiring minimum cell count enforcement for privacy +- `empty_is_na` — whether empty strings should be treated as NA (default: true) +- `description` — human-readable description of the column +- `references` — foreign key reference in `table.column` format -A namespaced YAML specification looks like this: +A fully-featured namespaced YAML specification looks like this: ```yaml version: "1.0" namespace: my_analysis: + prefix: my_analysis_ # database table prefix (defaults to {namespace}_) + description: "Results from the MyAnalysis package" tables: table_1: + description: "Primary analysis results" columns: - name: database_id type: varchar primary_key: true + description: "The database identifier" - name: analysis_id type: bigint primary_key: true @@ -55,23 +73,24 @@ namespace: type: varchar - name: domain_id type: varchar(20) - optional: true + nullable: true - name: start_day type: float - optional: true + nullable: true - name: end_day type: float - optional: true + nullable: true - name: is_binary type: varchar(1) - name: missing_means_zero type: varchar(1) - optional: true + nullable: true table_2: columns: - name: database_id type: varchar primary_key: true + references: table_1.database_id - name: analysis2_id type: bigint primary_key: true @@ -79,7 +98,7 @@ namespace: type: int - name: logic_description type: varchar - optional: true + nullable: true - name: valid_start_date type: Date - name: concept_name @@ -88,11 +107,38 @@ namespace: type: float ``` -The namespace name (e.g. `my_analysis`) becomes the table prefix in the database. -A table named `table_1` in namespace `my_analysis` will be created as `my_analysis_table_1`. +The namespace name (and `prefix`) determine the database table names. +A table `table_1` in namespace `my_analysis` with prefix `my_analysis_` will be created as +`my_analysis_table_1`. The `prefix` field is optional and defaults to `{namespace}_`. + +## Platform-specific configuration Platform-specific features such as table partitioning and indexes can be configured -per DBMS platform. See `?loadResultsDataModelFromYaml` for details. +per DBMS platform in the same YAML file: + +```yaml +platforms: + postgresql: + namespace: + my_analysis: + tables: + table_1: + partition_by: RANGE (analysis_id) + indexes: + - columns: [database_id, analysis_id] + unique: true + sql_server: + namespace: + my_analysis: + tables: + table_1: + indexes: + - columns: [database_id] + duckdb: {} + sqlite: {} +``` + +Features not supported by a platform (e.g., partitioning in duckdb) will emit a warning. ## CSV format (deprecated) @@ -112,15 +158,15 @@ to the YAML format. To load a specification file: ```{r eval = FALSE} -# YAML (recommended) +# YAML (recommended) - returns a data.frame spec <- ResultModelManager::loadResultsDataModelSpecifications("path/to/spec.yaml") -# The YAML specification is loaded via +# YAML with platform config - returns list(specification, platforms) result <- ResultModelManager::loadResultsDataModelFromYaml("path/to/spec.yaml") -spec <- result$specification # data.frame -platforms <- result$platforms # platform-specific config +spec <- result$specification # data.frame with namespace, tableName, ... +platforms <- result$platforms # platform-specific config -# CSV (deprecated) +# CSV (deprecated) - emits a deprecation warning spec <- ResultModelManager::loadResultsDataModelSpecifications("path/to/spec.csv") ``` @@ -146,9 +192,20 @@ sql <- ResultModelManager::generateSqlSchema(schemaDefinition = getResultsDataMo DatabaseConnector::renderTranslateExecuteSql(connection, sql, database_schema = "main", table_prefix = "pre_") DatabaseConnector::disconnect(connection) ``` + Note that the SQL generated by `generateSqlSchema` contains the required parameter `database_schema` and the optional parameter `table_prefix`. +If the specification was loaded from a YAML file (not a data.frame), you can pass a `platform` parameter +for platform-specific DDL (partitioning, indexes): + +```{r eval=FALSE} +sql <- ResultModelManager::generateSqlSchema( + csvFilepath = "path/to/spec.yaml", + platform = "postgresql" +) +``` + # Uploading results This section shows how to upload results conforming to the above specified schema. It is assumed that a zip file has been created with csv files corresponding to the table names provided in the diff --git a/vignettes/UsingQueryNamespaces.Rmd b/vignettes/UsingQueryNamespaces.Rmd index 56a065e..ef3191e 100644 --- a/vignettes/UsingQueryNamespaces.Rmd +++ b/vignettes/UsingQueryNamespaces.Rmd @@ -27,11 +27,17 @@ However, many find that writing SQL strings is often more convenient and portabl than `dplyr` calls allow. # Basic usage -The most basic usage is to create a specification with a single table that conforms to a valid data model specification +The most basic usage is to create a specification with a single table that conforms to a valid data model specification. +For packages that ship YAML specifications, the table names include a namespace prefix (see the +"Upload Functionality" vignette for the YAML schema format). + ```{r} library(ResultModelManager) +# Namespaced table spec (from a YAML file with namespace = "cg") tableSpecification <- data.frame( + namespace = "cg", + namespacePrefix = "cg_", tableName = "cohort_definition", columnName = c("cohort_definition_id", "cohort_name", "json", "sql"), primaryKey = c("yes", "no", "no", "no"), @@ -41,7 +47,7 @@ tableSpecification <- data.frame( Note, that generally we would save these tables to a yaml file that can be loaded (see the "Upload Functionality" vignette for the YAML schema format). -We then load a `QueryNamespace` instance with this table: +With a namespaced spec, the `QueryNamespace` registers tables as `@namespace_tableName`: ```{r} connectionDetails <- DatabaseConnector::createConnectionDetails("sqlite", server = tempfile()) @@ -54,20 +60,34 @@ qns <- createQueryNamespace( database_schema = "main" ) -# Create our schema within the namespace +# Create our schema within the namespace - table var is @cg_cohort_definition sql <- generateSqlSchema(schemaDefinition = tableSpecification) -# note - the table prefix and schema parameters are not neeeded +# note - the table prefix and schema parameters are not needed qns$executeSql(sql) ``` We can then query the table with sql that automatically replaces the table names: ```{r} -qns$queryDb("SELECT * FROM @database_schema.@cohort_definition") +qns$queryDb("SELECT * FROM @database_schema.@cg_cohort_definition") ``` -Note that the underlying query is already handling our `tablePrefix` for us, so we don't need to add it: +Note that the underlying query is already handling our `tablePrefix` for us, +so we don't need to add it. Here `@cg_cohort_definition` resolves to +`rwe_study_99_cg_cohort_definition`. + +## Non-namespaced table specs (backward compat) + +For backward compatibility, table specs without a `namespace` column work the same way +as before: ```{r} -qns$queryDb("SELECT * FROM @database_schema.@cohort_definition") +tableSpecification2 <- data.frame( + tableName = "cohort_definition", + columnName = c("cohort_definition_id", "cohort_name", "json", "sql"), + primaryKey = c("yes", "no", "no", "no"), + dataType = c("bigint", "varchar", "varchar", "varchar") +) +qns$addTableSpecification(tableSpecification2) +# Registers as @cohort_definition (no namespace prefix) ``` # Adding replacement variables at runtime From 0ceff3e4a67a306d02d09a1d87034ae6f2015400 Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 17:04:36 -0700 Subject: [PATCH 6/8] Fixed vignette --- vignettes/UsingQueryNamespaces.Rmd | 58 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/vignettes/UsingQueryNamespaces.Rmd b/vignettes/UsingQueryNamespaces.Rmd index ef3191e..1317200 100644 --- a/vignettes/UsingQueryNamespaces.Rmd +++ b/vignettes/UsingQueryNamespaces.Rmd @@ -26,10 +26,11 @@ This is not intended to replace usage of `dbplyr` style operations which are exp However, many find that writing SQL strings is often more convenient and portable to other programming language than `dplyr` calls allow. -# Basic usage -The most basic usage is to create a specification with a single table that conforms to a valid data model specification. -For packages that ship YAML specifications, the table names include a namespace prefix (see the -"Upload Functionality" vignette for the YAML schema format). +# Namespaced table specs + +For packages that ship YAML specifications, the table names include a namespace prefix +(see the "Upload Functionality" vignette for the YAML schema format). +With a namespaced spec, the `QueryNamespace` registers tables as `@namespace_tableName`: ```{r} library(ResultModelManager) @@ -43,13 +44,7 @@ tableSpecification <- data.frame( primaryKey = c("yes", "no", "no", "no"), dataType = c("bigint", "varchar", "varchar", "varchar") ) -``` -Note, that generally we would save these tables to a yaml file that can be loaded -(see the "Upload Functionality" vignette for the YAML schema format). - -With a namespaced spec, the `QueryNamespace` registers tables as `@namespace_tableName`: -```{r} connectionDetails <- DatabaseConnector::createConnectionDetails("sqlite", server = tempfile()) qns <- createQueryNamespace( connectionDetails = connectionDetails, @@ -60,7 +55,7 @@ qns <- createQueryNamespace( database_schema = "main" ) -# Create our schema within the namespace - table var is @cg_cohort_definition +# Create our schema within the namespace sql <- generateSqlSchema(schemaDefinition = tableSpecification) # note - the table prefix and schema parameters are not needed qns$executeSql(sql) @@ -74,26 +69,10 @@ Note that the underlying query is already handling our `tablePrefix` for us, so we don't need to add it. Here `@cg_cohort_definition` resolves to `rwe_study_99_cg_cohort_definition`. -## Non-namespaced table specs (backward compat) - -For backward compatibility, table specs without a `namespace` column work the same way -as before: - -```{r} -tableSpecification2 <- data.frame( - tableName = "cohort_definition", - columnName = c("cohort_definition_id", "cohort_name", "json", "sql"), - primaryKey = c("yes", "no", "no", "no"), - dataType = c("bigint", "varchar", "varchar", "varchar") -) -qns$addTableSpecification(tableSpecification2) -# Registers as @cohort_definition (no namespace prefix) -``` - # Adding replacement variables at runtime Variables can naturally be added at runtime, for example, in a query: ```{r} -qns$queryDb("SELECT * FROM @database_schema.@cohort_definition WHERE cohort_definition_id = @id", +qns$queryDb("SELECT * FROM @database_schema.@cg_cohort_definition WHERE cohort_definition_id = @id", id = 5 ) ``` @@ -107,15 +86,34 @@ Note that replacing the same variable will result in an error qns$addReplacementVariable("database_id", "my_cdm") ``` +# Non-namespaced table specs (backward compat) -We can also add to the table specification +For backward compatibility, table specs without a `namespace` column work the same way +as before, registering tables as `@tableName`: ```{r} +# Non-namespaced spec - separate QueryNamespace +connectionDetails2 <- DatabaseConnector::createConnectionDetails("sqlite", server = tempfile()) + tableSpecification2 <- data.frame( tableName = "database_info", columnName = c("database_id", "database_name"), primaryKey = c("yes", "no"), dataType = c("varchar", "varchar") ) -qns$addTableSpecification(tableSpecification2) + +qns2 <- createQueryNamespace( + connectionDetails = connectionDetails2, + tableSpecification = tableSpecification2, + tablePrefix = "legacy_", + snakeCaseToCamelCase = TRUE, + database_schema = "main" +) + +sql2 <- generateSqlSchema(schemaDefinition = tableSpecification2) +qns2$executeSql(sql2) +``` +Tables registered without a namespace are queried by their plain name: +```{r} +qns2$queryDb("SELECT * FROM @database_schema.@database_info") ``` From 18e09d1fd0441a1e50cb81c9fd8c877c11bdad8f Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 17:08:57 -0700 Subject: [PATCH 7/8] Missing docs --- DESCRIPTION | 2 +- NAMESPACE | 2 +- man/ConnectionHandler.Rd | 403 +++++++++++----------- man/DataMigrationManager.Rd | 284 ++++++++------- man/PooledConnectionHandler.Rd | 339 +++++++++--------- man/QueryNamespace.Rd | 339 +++++++++--------- man/ResultExportManager.Rd | 396 ++++++++++----------- man/ResultModelManager-package.Rd | 5 + man/csvToYaml.Rd | 31 ++ man/generateSqlSchema.Rd | 21 +- man/loadResultsDataModelFromYaml.Rd | 32 ++ man/loadResultsDataModelSpecifications.Rd | 2 +- 12 files changed, 992 insertions(+), 864 deletions(-) create mode 100644 man/csvToYaml.Rd create mode 100644 man/loadResultsDataModelFromYaml.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 8fe333f..e06e6ec 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -12,7 +12,6 @@ License: Apache License (== 2.0) Encoding: UTF-8 VignetteBuilder: knitr Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.2 Depends: R (>= 4.1.0), R6, @@ -46,3 +45,4 @@ Suggests: rJava, reticulate Config/testthat/edition: 3 +Config/roxygen2/version: 8.0.0 diff --git a/NAMESPACE b/NAMESPACE index ed5c17f..7fe1cbb 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,6 +7,7 @@ export(QueryNamespace) export(ResultExportManager) export(createQueryNamespace) export(createResultExportManager) +export(csvToYaml) export(deleteAllRowsForDatabaseId) export(deleteAllRowsForPrimaryKey) export(disablePythonUploads) @@ -15,7 +16,6 @@ export(generateSqlSchema) export(grantTablePermissions) export(install_psycopg2) export(loadResultsDataModelFromYaml) -export(csvToYaml) export(loadResultsDataModelSpecifications) export(pyPgUploadEnabled) export(pyUploadCsv) diff --git a/man/ConnectionHandler.Rd b/man/ConnectionHandler.Rd index 0cd92ec..ed8ef5f 100644 --- a/man/ConnectionHandler.Rd +++ b/man/ConnectionHandler.Rd @@ -3,297 +3,316 @@ \name{ConnectionHandler} \alias{ConnectionHandler} \title{ConnectionHandler} -\value{ -DatabaseConnector Connection instance -close Connection - -boolean TRUE if connection is valid -close Connection - -boolean TRUE if connection is valid -executeSql -} \description{ Class for handling DatabaseConnector:connection objects with consistent R6 interfaces for pooled and non-pooled connections. Allows a connection to cleanly be opened and closed and stored within class/object variables } \section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionDetails}}{DatabaseConnector connectionDetails object} + \if{html}{\out{
}} + \describe{ + \item{\code{connectionDetails}}{DatabaseConnector connectionDetails object} -\item{\code{con}}{DatabaseConnector connection object} + \item{\code{con}}{DatabaseConnector connection object} -\item{\code{isActive}}{Is connection active or not#'} + \item{\code{isActive}}{Is connection active or not#'} -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} -} -\if{html}{\out{
}} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} + } + \if{html}{\out{
}} } \section{Methods}{ \subsection{Public methods}{ -\itemize{ -\item \href{#method-ConnectionHandler-new}{\code{ConnectionHandler$new()}} -\item \href{#method-ConnectionHandler-dbms}{\code{ConnectionHandler$dbms()}} -\item \href{#method-ConnectionHandler-tbl}{\code{ConnectionHandler$tbl()}} -\item \href{#method-ConnectionHandler-renderTranslateSql}{\code{ConnectionHandler$renderTranslateSql()}} -\item \href{#method-ConnectionHandler-initConnection}{\code{ConnectionHandler$initConnection()}} -\item \href{#method-ConnectionHandler-getConnection}{\code{ConnectionHandler$getConnection()}} -\item \href{#method-ConnectionHandler-closeConnection}{\code{ConnectionHandler$closeConnection()}} -\item \href{#method-ConnectionHandler-dbIsValid}{\code{ConnectionHandler$dbIsValid()}} -\item \href{#method-ConnectionHandler-finalize}{\code{ConnectionHandler$finalize()}} -\item \href{#method-ConnectionHandler-queryDb}{\code{ConnectionHandler$queryDb()}} -\item \href{#method-ConnectionHandler-executeSql}{\code{ConnectionHandler$executeSql()}} -\item \href{#method-ConnectionHandler-queryFunction}{\code{ConnectionHandler$queryFunction()}} -\item \href{#method-ConnectionHandler-executeFunction}{\code{ConnectionHandler$executeFunction()}} -\item \href{#method-ConnectionHandler-clone}{\code{ConnectionHandler$clone()}} -} + \itemize{ + \item \href{#method-ConnectionHandler-initialize}{\code{ConnectionHandler$new()}} + \item \href{#method-ConnectionHandler-dbms}{\code{ConnectionHandler$dbms()}} + \item \href{#method-ConnectionHandler-tbl}{\code{ConnectionHandler$tbl()}} + \item \href{#method-ConnectionHandler-renderTranslateSql}{\code{ConnectionHandler$renderTranslateSql()}} + \item \href{#method-ConnectionHandler-initConnection}{\code{ConnectionHandler$initConnection()}} + \item \href{#method-ConnectionHandler-getConnection}{\code{ConnectionHandler$getConnection()}} + \item \href{#method-ConnectionHandler-closeConnection}{\code{ConnectionHandler$closeConnection()}} + \item \href{#method-ConnectionHandler-dbIsValid}{\code{ConnectionHandler$dbIsValid()}} + \item \href{#method-ConnectionHandler-finalize}{\code{ConnectionHandler$finalize()}} + \item \href{#method-ConnectionHandler-queryDb}{\code{ConnectionHandler$queryDb()}} + \item \href{#method-ConnectionHandler-executeSql}{\code{ConnectionHandler$executeSql()}} + \item \href{#method-ConnectionHandler-queryFunction}{\code{ConnectionHandler$queryFunction()}} + \item \href{#method-ConnectionHandler-executeFunction}{\code{ConnectionHandler$executeFunction()}} + \item \href{#method-ConnectionHandler-clone}{\code{ConnectionHandler$clone()}} + } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ConnectionHandler-new}{}}} -\subsection{Method \code{new()}}{ -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$new( +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ConnectionHandler-initialize}{}}} +\subsection{\code{ConnectionHandler$new()}}{ + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$new( connectionDetails, loadConnection = TRUE, snakeCaseToCamelCase = TRUE -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionDetails}}{DatabaseConnector::connectionDetails class} - -\item{\code{loadConnection}}{Boolean option to load connection right away} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default) +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connectionDetails}}{DatabaseConnector::connectionDetails class} + \item{\code{loadConnection}}{Boolean option to load connection right away} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default) get dbms} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-dbms}{}}} -\subsection{Method \code{dbms()}}{ -Get the dbms type of the connection +\subsection{\code{ConnectionHandler$dbms()}}{ + Get the dbms type of the connection get table -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$dbms()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$dbms()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-tbl}{}}} -\subsection{Method \code{tbl()}}{ -get a dplyr table object (i.e. lazy loaded) -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$tbl(table, databaseSchema = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{table}}{table name} - -\item{\code{databaseSchema}}{databaseSchema to which table belongs +\subsection{\code{ConnectionHandler$tbl()}}{ + get a dplyr table object (i.e. lazy loaded) + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$tbl(table, databaseSchema = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{table}}{table name} + \item{\code{databaseSchema}}{databaseSchema to which table belongs Render Translate Sql.} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-renderTranslateSql}{}}} -\subsection{Method \code{renderTranslateSql()}}{ -Masked call to SqlRender -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$renderTranslateSql(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{Sql query string} - -\item{\code{...}}{Elipsis +\subsection{\code{ConnectionHandler$renderTranslateSql()}}{ + Masked call to SqlRender + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$renderTranslateSql(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{Sql query string} + \item{\code{...}}{Elipsis initConnection} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-initConnection}{}}} -\subsection{Method \code{initConnection()}}{ -Load connection +\subsection{\code{ConnectionHandler$initConnection()}}{ + Load connection Get Connection -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$initConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$initConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-getConnection}{}}} -\subsection{Method \code{getConnection()}}{ -Returns connection for use with standard DatabaseConnector calls. +\subsection{\code{ConnectionHandler$getConnection()}}{ + Returns connection for use with standard DatabaseConnector calls. Connects automatically if it isn't yet loaded -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$getConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$getConnection()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + DatabaseConnector Connection instance +close Connection + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-closeConnection}{}}} -\subsection{Method \code{closeConnection()}}{ -Closes connection (if active) +\subsection{\code{ConnectionHandler$closeConnection()}}{ + Closes connection (if active) db Is Valid -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$closeConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$closeConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-dbIsValid}{}}} -\subsection{Method \code{dbIsValid()}}{ -Masks call to DBI::dbIsValid. Returns False if connection is NULL -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$dbIsValid()}\if{html}{\out{
}} +\subsection{\code{ConnectionHandler$dbIsValid()}}{ + Masks call to DBI::dbIsValid. Returns False if connection is NULL + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$dbIsValid()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + boolean TRUE if connection is valid +close Connection + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-finalize}{}}} -\subsection{Method \code{finalize()}}{ -Closes connection (if active) +\subsection{\code{ConnectionHandler$finalize()}}{ + Closes connection (if active) queryDb -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$finalize()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$finalize()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-queryDb}{}}} -\subsection{Method \code{queryDb()}}{ -query database and return the resulting data.frame +\subsection{\code{ConnectionHandler$queryDb()}}{ + query database and return the resulting data.frame If environment variable LIMIT_ROW_COUNT is set Returned rows are limited to this value (no default) Limit row count is intended for web applications that may cause a denial of service if they consume too many resources. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$queryDb( + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$queryDb( sql, snakeCaseToCamelCase = self$snakeCaseToCamelCase, overrideRowLimit = FALSE, ... -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} - -\item{\code{overrideRowLimit}}{(Optional) Boolean. In some cases, where row limit is enforced on the system +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} + \item{\code{overrideRowLimit}}{(Optional) Boolean. In some cases, where row limit is enforced on the system You may wish to ignore it.} - -\item{\code{...}}{Additional query parameters} -} -\if{html}{\out{
}} -} + \item{\code{...}}{Additional query parameters} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + boolean TRUE if connection is valid +executeSql + } } + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-executeSql}{}}} -\subsection{Method \code{executeSql()}}{ -execute set of database queries -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$executeSql(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{...}}{Additional query parameters +\subsection{\code{ConnectionHandler$executeSql()}}{ + execute set of database queries + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$executeSql(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{...}}{Additional query parameters query Function} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-queryFunction}{}}} -\subsection{Method \code{queryFunction()}}{ -queryFunction that can be overriden with subclasses (e.g. use different base function or intercept query) +\subsection{\code{ConnectionHandler$queryFunction()}}{ + queryFunction that can be overriden with subclasses (e.g. use different base function or intercept query) Does not translate or render sql. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$queryFunction( + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$queryFunction( sql, snakeCaseToCamelCase = self$snakeCaseToCamelCase, connection = self$getConnection() -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} - -\item{\code{connection}}{(Optional) connection object +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} + \item{\code{connection}}{(Optional) connection object execute Function} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-executeFunction}{}}} -\subsection{Method \code{executeFunction()}}{ -exec query Function that can be overriden with subclasses (e.g. use different base function or intercept query) +\subsection{\code{ConnectionHandler$executeFunction()}}{ + exec query Function that can be overriden with subclasses (e.g. use different base function or intercept query) Does not translate or render sql. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$executeFunction(sql, connection = self$getConnection())}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$executeFunction(sql, connection = self$getConnection())} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{connection}}{connection object} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{connection}}{connection object} -} -\if{html}{\out{
}} -} -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ConnectionHandler-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ConnectionHandler$clone(deep = FALSE)}\if{html}{\out{
}} +\subsection{\code{ConnectionHandler$clone()}}{ + The objects of this class are cloneable with this method. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ConnectionHandler$clone(deep = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{deep}}{Whether to make a deep clone.} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} } diff --git a/man/DataMigrationManager.Rd b/man/DataMigrationManager.Rd index a48d8ab..b5cbd8b 100644 --- a/man/DataMigrationManager.Rd +++ b/man/DataMigrationManager.Rd @@ -3,10 +3,6 @@ \name{DataMigrationManager} \alias{DataMigrationManager} \title{DataMigrationManager (DMM)} -\value{ -data frame all migrations, including file name, order and execution status -Get connection handler -} \description{ R6 class for management of database migration } @@ -14,42 +10,43 @@ R6 class for management of database migration \link{ConnectionHandler} for information on returned class } \section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{migrationPath}}{Path migrations exist in} + \if{html}{\out{
}} + \describe{ + \item{\code{migrationPath}}{Path migrations exist in} -\item{\code{databaseSchema}}{Path migrations exist in} + \item{\code{databaseSchema}}{Path migrations exist in} -\item{\code{packageName}}{packageName, can be null} + \item{\code{packageName}}{packageName, can be null} -\item{\code{tablePrefix}}{tablePrefix, can be empty character vector} + \item{\code{tablePrefix}}{tablePrefix, can be empty character vector} -\item{\code{packageTablePrefix}}{packageTablePrefix, can be empty character vector} -} -\if{html}{\out{
}} + \item{\code{packageTablePrefix}}{packageTablePrefix, can be empty character vector} + } + \if{html}{\out{
}} } \section{Methods}{ \subsection{Public methods}{ -\itemize{ -\item \href{#method-DataMigrationManager-new}{\code{DataMigrationManager$new()}} -\item \href{#method-DataMigrationManager-migrationTableExists}{\code{DataMigrationManager$migrationTableExists()}} -\item \href{#method-DataMigrationManager-getMigrationsPath}{\code{DataMigrationManager$getMigrationsPath()}} -\item \href{#method-DataMigrationManager-getStatus}{\code{DataMigrationManager$getStatus()}} -\item \href{#method-DataMigrationManager-getConnectionHandler}{\code{DataMigrationManager$getConnectionHandler()}} -\item \href{#method-DataMigrationManager-check}{\code{DataMigrationManager$check()}} -\item \href{#method-DataMigrationManager-executeMigrations}{\code{DataMigrationManager$executeMigrations()}} -\item \href{#method-DataMigrationManager-closeConnection}{\code{DataMigrationManager$closeConnection()}} -\item \href{#method-DataMigrationManager-isPackage}{\code{DataMigrationManager$isPackage()}} -\item \href{#method-DataMigrationManager-finalize}{\code{DataMigrationManager$finalize()}} -\item \href{#method-DataMigrationManager-clone}{\code{DataMigrationManager$clone()}} -} + \itemize{ + \item \href{#method-DataMigrationManager-initialize}{\code{DataMigrationManager$new()}} + \item \href{#method-DataMigrationManager-migrationTableExists}{\code{DataMigrationManager$migrationTableExists()}} + \item \href{#method-DataMigrationManager-getMigrationsPath}{\code{DataMigrationManager$getMigrationsPath()}} + \item \href{#method-DataMigrationManager-getStatus}{\code{DataMigrationManager$getStatus()}} + \item \href{#method-DataMigrationManager-getConnectionHandler}{\code{DataMigrationManager$getConnectionHandler()}} + \item \href{#method-DataMigrationManager-check}{\code{DataMigrationManager$check()}} + \item \href{#method-DataMigrationManager-executeMigrations}{\code{DataMigrationManager$executeMigrations()}} + \item \href{#method-DataMigrationManager-closeConnection}{\code{DataMigrationManager$closeConnection()}} + \item \href{#method-DataMigrationManager-isPackage}{\code{DataMigrationManager$isPackage()}} + \item \href{#method-DataMigrationManager-finalize}{\code{DataMigrationManager$finalize()}} + \item \href{#method-DataMigrationManager-clone}{\code{DataMigrationManager$clone()}} + } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-DataMigrationManager-new}{}}} -\subsection{Method \code{new()}}{ -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$new( +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-DataMigrationManager-initialize}{}}} +\subsection{\code{DataMigrationManager$new()}}{ + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$new( connectionDetails, databaseSchema, tablePrefix = "", @@ -57,167 +54,186 @@ R6 class for management of database migration migrationPath, packageName = NULL, migrationRegexp = .defaultMigrationRegexp -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionDetails}}{DatabaseConnector connection details object} - -\item{\code{databaseSchema}}{Database Schema to execute on} - -\item{\code{tablePrefix}}{Optional table prefix for all tables (e.g. plp, cm, cd etc)} - -\item{\code{packageTablePrefix}}{A table prefix when used in conjunction with other package results schema, +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connectionDetails}}{DatabaseConnector connection details object} + \item{\code{databaseSchema}}{Database Schema to execute on} + \item{\code{tablePrefix}}{Optional table prefix for all tables (e.g. plp, cm, cd etc)} + \item{\code{packageTablePrefix}}{A table prefix when used in conjunction with other package results schema, e.g. "cd_", "sccs_", "plp_", "cm_"} - -\item{\code{migrationPath}}{Path to location of migration sql files. If in package mode, this should just + \item{\code{migrationPath}}{Path to location of migration sql files. If in package mode, this should just be a folder (e.g. "migrations") that lives in the location "sql/sql_server" (and) other database platforms. If in folder model, the folder must include "sql_server" in the relative path, (e.g if migrationPath = 'migrations' then the folder 'migrations/sql_server' should exists)} - -\item{\code{packageName}}{If in package mode, the name of the R package} - -\item{\code{migrationRegexp}}{(Optional) regular expression pattern default is \verb{(Migration_([0-9]+))-(.+).sql} + \item{\code{packageName}}{If in package mode, the name of the R package} + \item{\code{migrationRegexp}}{(Optional) regular expression pattern default is \verb{(Migration_([0-9]+))-(.+).sql} Migration table exists} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-migrationTableExists}{}}} -\subsection{Method \code{migrationTableExists()}}{ -Check if migration table is present in schema -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$migrationTableExists()}\if{html}{\out{
}} -} - -\subsection{Returns}{ -boolean +\subsection{\code{DataMigrationManager$migrationTableExists()}}{ + Check if migration table is present in schema + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$migrationTableExists()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + boolean Get path of migrations + } } -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-getMigrationsPath}{}}} -\subsection{Method \code{getMigrationsPath()}}{ -Get path to sql migration files -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$getMigrationsPath(dbms = "sql server")}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{dbms}}{Optionally specify the dbms that the migration fits under +\subsection{\code{DataMigrationManager$getMigrationsPath()}}{ + Get path to sql migration files + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$getMigrationsPath(dbms = "sql server")} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{dbms}}{Optionally specify the dbms that the migration fits under Get status of result model} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-getStatus}{}}} -\subsection{Method \code{getStatus()}}{ -Get status of all migrations (executed or not) -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$getStatus()}\if{html}{\out{
}} +\subsection{\code{DataMigrationManager$getStatus()}}{ + Get status of all migrations (executed or not) + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$getStatus()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + data frame all migrations, including file name, order and execution status +Get connection handler + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-getConnectionHandler}{}}} -\subsection{Method \code{getConnectionHandler()}}{ -Return connection handler instance -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$getConnectionHandler()}\if{html}{\out{
}} -} - -\subsection{Returns}{ -ConnectionHandler instance +\subsection{\code{DataMigrationManager$getConnectionHandler()}}{ + Return connection handler instance + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$getConnectionHandler()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + ConnectionHandler instance Check migrations in folder + } } -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-check}{}}} -\subsection{Method \code{check()}}{ -Check if file names are valid for migrations +\subsection{\code{DataMigrationManager$check()}}{ + Check if file names are valid for migrations Execute Migrations -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$check()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$check()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-executeMigrations}{}}} -\subsection{Method \code{executeMigrations()}}{ -Execute any unexecuted migrations -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$executeMigrations(stopMigrationVersion = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{stopMigrationVersion}}{(Optional) Migrate to a specific migration number +\subsection{\code{DataMigrationManager$executeMigrations()}}{ + Execute any unexecuted migrations + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$executeMigrations(stopMigrationVersion = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{stopMigrationVersion}}{(Optional) Migrate to a specific migration number closeConnection} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-closeConnection}{}}} -\subsection{Method \code{closeConnection()}}{ -close connection, if active +\subsection{\code{DataMigrationManager$closeConnection()}}{ + close connection, if active isPackage -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$closeConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$closeConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-isPackage}{}}} -\subsection{Method \code{isPackage()}}{ -is a package folder structure or not +\subsection{\code{DataMigrationManager$isPackage()}}{ + is a package folder structure or not finalize -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$isPackage()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$isPackage()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-finalize}{}}} -\subsection{Method \code{finalize()}}{ -Deprecated call, will be removed in a future version -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$finalize()}\if{html}{\out{
}} +\subsection{\code{DataMigrationManager$finalize()}}{ + Deprecated call, will be removed in a future version + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$finalize()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-DataMigrationManager-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{DataMigrationManager$clone(deep = FALSE)}\if{html}{\out{
}} +\subsection{\code{DataMigrationManager$clone()}}{ + The objects of this class are cloneable with this method. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{DataMigrationManager$clone(deep = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{deep}}{Whether to make a deep clone.} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} } diff --git a/man/PooledConnectionHandler.Rd b/man/PooledConnectionHandler.Rd index 0a6d783..dcbec7e 100644 --- a/man/PooledConnectionHandler.Rd +++ b/man/PooledConnectionHandler.Rd @@ -3,262 +3,263 @@ \name{PooledConnectionHandler} \alias{PooledConnectionHandler} \title{Pooled Connection Handler} -\value{ -boolean TRUE if connection is valid -executeSql -} \description{ Transparently works the same way as a standard connection handler but stores pooled connections. Useful for long running applications that serve multiple concurrent requests. Note that a side effect of using this is that each call to this increments the .GlobalEnv attribute \code{RMMPooledHandlerCount} } \section{Super class}{ -\code{\link[ResultModelManager:ConnectionHandler]{ResultModelManager::ConnectionHandler}} -> \code{PooledConnectionHandler} +\code{\link[ResultModelManager:ConnectionHandler]{ConnectionHandler}} -> \code{PooledConnectionHandler} } \section{Methods}{ \subsection{Public methods}{ -\itemize{ -\item \href{#method-PooledConnectionHandler-new}{\code{PooledConnectionHandler$new()}} -\item \href{#method-PooledConnectionHandler-initConnection}{\code{PooledConnectionHandler$initConnection()}} -\item \href{#method-PooledConnectionHandler-getCheckedOutConnectionPath}{\code{PooledConnectionHandler$getCheckedOutConnectionPath()}} -\item \href{#method-PooledConnectionHandler-getConnection}{\code{PooledConnectionHandler$getConnection()}} -\item \href{#method-PooledConnectionHandler-dbms}{\code{PooledConnectionHandler$dbms()}} -\item \href{#method-PooledConnectionHandler-closeConnection}{\code{PooledConnectionHandler$closeConnection()}} -\item \href{#method-PooledConnectionHandler-queryDb}{\code{PooledConnectionHandler$queryDb()}} -\item \href{#method-PooledConnectionHandler-executeSql}{\code{PooledConnectionHandler$executeSql()}} -\item \href{#method-PooledConnectionHandler-queryFunction}{\code{PooledConnectionHandler$queryFunction()}} -\item \href{#method-PooledConnectionHandler-executeFunction}{\code{PooledConnectionHandler$executeFunction()}} -\item \href{#method-PooledConnectionHandler-clone}{\code{PooledConnectionHandler$clone()}} -} -} -\if{html}{\out{ -
Inherited methods + \itemize{ + \item \href{#method-PooledConnectionHandler-initialize}{\code{PooledConnectionHandler$new()}} + \item \href{#method-PooledConnectionHandler-initConnection}{\code{PooledConnectionHandler$initConnection()}} + \item \href{#method-PooledConnectionHandler-getCheckedOutConnectionPath}{\code{PooledConnectionHandler$getCheckedOutConnectionPath()}} + \item \href{#method-PooledConnectionHandler-getConnection}{\code{PooledConnectionHandler$getConnection()}} + \item \href{#method-PooledConnectionHandler-dbms}{\code{PooledConnectionHandler$dbms()}} + \item \href{#method-PooledConnectionHandler-closeConnection}{\code{PooledConnectionHandler$closeConnection()}} + \item \href{#method-PooledConnectionHandler-queryDb}{\code{PooledConnectionHandler$queryDb()}} + \item \href{#method-PooledConnectionHandler-executeSql}{\code{PooledConnectionHandler$executeSql()}} + \item \href{#method-PooledConnectionHandler-queryFunction}{\code{PooledConnectionHandler$queryFunction()}} + \item \href{#method-PooledConnectionHandler-executeFunction}{\code{PooledConnectionHandler$executeFunction()}} + \item \href{#method-PooledConnectionHandler-clone}{\code{PooledConnectionHandler$clone()}} + } +} +\if{html}{\out{
Inherited methods -
-}} +
}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-PooledConnectionHandler-new}{}}} -\subsection{Method \code{new()}}{ -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$new( +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-PooledConnectionHandler-initialize}{}}} +\subsection{\code{PooledConnectionHandler$new()}}{ + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$new( connectionDetails = NULL, snakeCaseToCamelCase = TRUE, loadConnection = TRUE, dbConnectArgs = NULL, forceJdbcConnection = TRUE -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionDetails}}{DatabaseConnector::connectionDetails class} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} - -\item{\code{loadConnection}}{Boolean option to load connection right away} - -\item{\code{dbConnectArgs}}{Optional arguments to call pool::dbPool overrides default usage of connectionDetails} - -\item{\code{forceJdbcConnection}}{Force JDBC connection (requires using DatabaseConnector ConnectionDetails) +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connectionDetails}}{DatabaseConnector::connectionDetails class} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} + \item{\code{loadConnection}}{Boolean option to load connection right away} + \item{\code{dbConnectArgs}}{Optional arguments to call pool::dbPool overrides default usage of connectionDetails} + \item{\code{forceJdbcConnection}}{Force JDBC connection (requires using DatabaseConnector ConnectionDetails) initialize pooled db connection} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-initConnection}{}}} -\subsection{Method \code{initConnection()}}{ -Overrides ConnectionHandler Call +\subsection{\code{PooledConnectionHandler$initConnection()}}{ + Overrides ConnectionHandler Call Used for getting a checked out connection from a given environment (if one exists) -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$initConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$initConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-getCheckedOutConnectionPath}{}}} -\subsection{Method \code{getCheckedOutConnectionPath()}}{ -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$getCheckedOutConnectionPath()}\if{html}{\out{
}} +\subsection{\code{PooledConnectionHandler$getCheckedOutConnectionPath()}}{ + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$getCheckedOutConnectionPath()} + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{.deferedFrame}}{defaults to the parent frame of the calling block. -Get Connection} -} -\if{html}{\out{
}} -} -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-getConnection}{}}} -\subsection{Method \code{getConnection()}}{ -Returns a connection from the pool +\subsection{\code{PooledConnectionHandler$getConnection()}}{ + Returns a connection from the pool When the desired frame exits, the connection will be returned to the pool As a side effect, the connection is stored as an attribute within the calling frame (e.g. the same function) to prevent multiple connections being spawned, which limits performance. If you call this somewhere you need to think about returning the object or you may create a connection that is never returned to the pool. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$getConnection(.deferedFrame = parent.frame(n = 2))}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{.deferedFrame}}{defaults to the parent frame of the calling block. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$getConnection(.deferedFrame = parent.frame(n = 2))} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{.deferedFrame}}{defaults to the parent frame of the calling block. get dbms} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-dbms}{}}} -\subsection{Method \code{dbms()}}{ -Get the dbms type of the connection +\subsection{\code{PooledConnectionHandler$dbms()}}{ + Get the dbms type of the connection Close Connection -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$dbms()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$dbms()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-closeConnection}{}}} -\subsection{Method \code{closeConnection()}}{ -Overrides ConnectionHandler Call - closes all active connections called with getConnection +\subsection{\code{PooledConnectionHandler$closeConnection()}}{ + Overrides ConnectionHandler Call - closes all active connections called with getConnection queryDb -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$closeConnection()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$closeConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-queryDb}{}}} -\subsection{Method \code{queryDb()}}{ -query database and return the resulting data.frame +\subsection{\code{PooledConnectionHandler$queryDb()}}{ + query database and return the resulting data.frame If environment variable LIMIT_ROW_COUNT is set Returned rows are limited to this value (no default) Limit row count is intended for web applications that may cause a denial of service if they consume too many resources. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$queryDb( + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$queryDb( sql, snakeCaseToCamelCase = self$snakeCaseToCamelCase, overrideRowLimit = FALSE, ... -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} - -\item{\code{overrideRowLimit}}{(Optional) Boolean. In some cases, where row limit is enforced on the system +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default)} + \item{\code{overrideRowLimit}}{(Optional) Boolean. In some cases, where row limit is enforced on the system You may wish to ignore it.} - -\item{\code{...}}{Additional query parameters} -} -\if{html}{\out{
}} -} + \item{\code{...}}{Additional query parameters} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + boolean TRUE if connection is valid +executeSql + } } + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-executeSql}{}}} -\subsection{Method \code{executeSql()}}{ -execute set of database queries -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$executeSql(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{...}}{Additional query parameters +\subsection{\code{PooledConnectionHandler$executeSql()}}{ + execute set of database queries + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$executeSql(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{...}}{Additional query parameters query Function} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-queryFunction}{}}} -\subsection{Method \code{queryFunction()}}{ -Overrides ConnectionHandler Call. Does not translate or render sql. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$queryFunction( +\subsection{\code{PooledConnectionHandler$queryFunction()}}{ + Overrides ConnectionHandler Call. Does not translate or render sql. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$queryFunction( sql, snakeCaseToCamelCase = self$snakeCaseToCamelCase, connection -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default) +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{snakeCaseToCamelCase}}{(Optional) Boolean. return the results columns in camel case (default) query Function} - -\item{\code{connection}}{db connection assumes pooling is handled outside of call} -} -\if{html}{\out{
}} -} + \item{\code{connection}}{db connection assumes pooling is handled outside of call} + } + \if{html}{\out{
}} + } } + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-executeFunction}{}}} -\subsection{Method \code{executeFunction()}}{ -Overrides ConnectionHandler Call. Does not translate or render sql. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$executeFunction(sql, connection)}\if{html}{\out{
}} +\subsection{\code{PooledConnectionHandler$executeFunction()}}{ + Overrides ConnectionHandler Call. Does not translate or render sql. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$executeFunction(sql, connection)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{sql query string} + \item{\code{connection}}{DatabaseConnector connection. Assumes pooling is handled outside of call} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{sql query string} - -\item{\code{connection}}{DatabaseConnector connection. Assumes pooling is handled outside of call} -} -\if{html}{\out{
}} -} -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-PooledConnectionHandler-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{PooledConnectionHandler$clone(deep = FALSE)}\if{html}{\out{
}} +\subsection{\code{PooledConnectionHandler$clone()}}{ + The objects of this class are cloneable with this method. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{PooledConnectionHandler$clone(deep = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{deep}}{Whether to make a deep clone.} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} } diff --git a/man/QueryNamespace.Rd b/man/QueryNamespace.Rd index 631fe75..18bb2bd 100644 --- a/man/QueryNamespace.Rd +++ b/man/QueryNamespace.Rd @@ -61,233 +61,246 @@ unlink("test_db.sqlite") } \section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{tablePrefix}}{tablePrefix to use} -} -\if{html}{\out{
}} + \if{html}{\out{
}} + \describe{ + \item{\code{tablePrefix}}{tablePrefix to use} + } + \if{html}{\out{
}} } \section{Methods}{ \subsection{Public methods}{ -\itemize{ -\item \href{#method-QueryNamespace-new}{\code{QueryNamespace$new()}} -\item \href{#method-QueryNamespace-setConnectionHandler}{\code{QueryNamespace$setConnectionHandler()}} -\item \href{#method-QueryNamespace-getConnectionHandler}{\code{QueryNamespace$getConnectionHandler()}} -\item \href{#method-QueryNamespace-addReplacementVariable}{\code{QueryNamespace$addReplacementVariable()}} -\item \href{#method-QueryNamespace-addTableSpecification}{\code{QueryNamespace$addTableSpecification()}} -\item \href{#method-QueryNamespace-render}{\code{QueryNamespace$render()}} -\item \href{#method-QueryNamespace-queryDb}{\code{QueryNamespace$queryDb()}} -\item \href{#method-QueryNamespace-executeSql}{\code{QueryNamespace$executeSql()}} -\item \href{#method-QueryNamespace-getVars}{\code{QueryNamespace$getVars()}} -\item \href{#method-QueryNamespace-closeConnection}{\code{QueryNamespace$closeConnection()}} -\item \href{#method-QueryNamespace-clone}{\code{QueryNamespace$clone()}} -} + \itemize{ + \item \href{#method-QueryNamespace-initialize}{\code{QueryNamespace$new()}} + \item \href{#method-QueryNamespace-setConnectionHandler}{\code{QueryNamespace$setConnectionHandler()}} + \item \href{#method-QueryNamespace-getConnectionHandler}{\code{QueryNamespace$getConnectionHandler()}} + \item \href{#method-QueryNamespace-addReplacementVariable}{\code{QueryNamespace$addReplacementVariable()}} + \item \href{#method-QueryNamespace-addTableSpecification}{\code{QueryNamespace$addTableSpecification()}} + \item \href{#method-QueryNamespace-render}{\code{QueryNamespace$render()}} + \item \href{#method-QueryNamespace-queryDb}{\code{QueryNamespace$queryDb()}} + \item \href{#method-QueryNamespace-executeSql}{\code{QueryNamespace$executeSql()}} + \item \href{#method-QueryNamespace-getVars}{\code{QueryNamespace$getVars()}} + \item \href{#method-QueryNamespace-closeConnection}{\code{QueryNamespace$closeConnection()}} + \item \href{#method-QueryNamespace-clone}{\code{QueryNamespace$clone()}} + } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-QueryNamespace-new}{}}} -\subsection{Method \code{new()}}{ -initialize class -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$new( +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-QueryNamespace-initialize}{}}} +\subsection{\code{QueryNamespace$new()}}{ + initialize class + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$new( connectionHandler = NULL, tableSpecification = NULL, tablePrefix = "", ... -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionHandler}}{ConnectionHandler instance @seealso\link{ConnectionHandler}} - -\item{\code{tableSpecification}}{tableSpecification data.frame} - -\item{\code{tablePrefix}}{constant string to prefix all tables with} - -\item{\code{...}}{additional replacement variables e.g. database_schema, vocabulary_schema etc +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connectionHandler}}{ConnectionHandler instance @seealso\link{ConnectionHandler}} + \item{\code{tableSpecification}}{tableSpecification data.frame} + \item{\code{tablePrefix}}{constant string to prefix all tables with} + \item{\code{...}}{additional replacement variables e.g. database_schema, vocabulary_schema etc Set Connection Handler} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-setConnectionHandler}{}}} -\subsection{Method \code{setConnectionHandler()}}{ -set connection handler object for object -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$setConnectionHandler(connectionHandler)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connectionHandler}}{ConnectionHandler instance +\subsection{\code{QueryNamespace$setConnectionHandler()}}{ + set connection handler object for object + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$setConnectionHandler(connectionHandler)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connectionHandler}}{ConnectionHandler instance Get connection handler} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-getConnectionHandler}{}}} -\subsection{Method \code{getConnectionHandler()}}{ -get connection handler obeject or throw error if not set -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$getConnectionHandler()}\if{html}{\out{
}} +\subsection{\code{QueryNamespace$getConnectionHandler()}}{ + get connection handler obeject or throw error if not set + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$getConnectionHandler()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-addReplacementVariable}{}}} -\subsection{Method \code{addReplacementVariable()}}{ -add a variable to automatically be replaced in query strings (e.g. @database_schema.@table_name becomes 'database_schema.table_1') -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$addReplacementVariable(key, value, replace = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{key}}{variable name string (without @) to be replaced, eg. "table_name"} - -\item{\code{value}}{atomic value for replacement} - -\item{\code{replace}}{if a variable of the same key is found, overwrite it +\subsection{\code{QueryNamespace$addReplacementVariable()}}{ + add a variable to automatically be replaced in query strings (e.g. @database_schema.@table_name becomes 'database_schema.table_1') + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$addReplacementVariable(key, value, replace = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{variable name string (without @) to be replaced, eg. "table_name"} + \item{\code{value}}{atomic value for replacement} + \item{\code{replace}}{if a variable of the same key is found, overwrite it add table specification} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-addTableSpecification}{}}} -\subsection{Method \code{addTableSpecification()}}{ -add a variable to automatically be replaced in query strings (e.g. @database_schema.@table_name becomes -'database_schema.table_1') -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$addTableSpecification( +\subsection{\code{QueryNamespace$addTableSpecification()}}{ + add a variable to automatically be replaced in query strings (e.g. @database_schema.@table_name becomes +'database_schema.table_1'). If the specification data.frame contains a 'namespace' column, table names +will be registered as '@namespace_tableName' and map to '{tablePrefix}{namespace}_{tableName}'. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$addTableSpecification( tableSpecification, useTablePrefix = TRUE, tablePrefix = self$tablePrefix, replace = TRUE -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{tableSpecification}}{table specification data.frame conforming to column names tableName, columnName, dataType and primaryKey} - -\item{\code{useTablePrefix}}{prefix the results with the tablePrefix (TRUE)} - -\item{\code{tablePrefix}}{prefix string - defaults to class variable set during initialization} - -\item{\code{replace}}{replace existing variables of the same name +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{tableSpecification}}{table specification data.frame conforming to column names tableName, columnName, dataType and primaryKey. +May optionally include a 'namespace' column for namespace-aware registration.} + \item{\code{useTablePrefix}}{prefix the results with the tablePrefix (TRUE)} + \item{\code{tablePrefix}}{prefix string - defaults to class variable set during initialization} + \item{\code{replace}}{replace existing variables of the same name Render} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-render}{}}} -\subsection{Method \code{render()}}{ -Call to SqlRender::render replacing names stored in this class -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$render(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{query string} - -\item{\code{...}}{additional variables to be passed to SqlRender::render - will overwrite anything in namespace +\subsection{\code{QueryNamespace$render()}}{ + Call to SqlRender::render replacing names stored in this class + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$render(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{query string} + \item{\code{...}}{additional variables to be passed to SqlRender::render - will overwrite anything in namespace query Sql} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-queryDb}{}}} -\subsection{Method \code{queryDb()}}{ -Call to -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$queryDb(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{query string} - -\item{\code{...}}{additional variables to send to SqlRender::render +\subsection{\code{QueryNamespace$queryDb()}}{ + Call to + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$queryDb(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{query string} + \item{\code{...}}{additional variables to send to SqlRender::render execute Sql} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-executeSql}{}}} -\subsection{Method \code{executeSql()}}{ -Call to execute sql within namespaced queries -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$executeSql(sql, ...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{sql}}{query string} - -\item{\code{...}}{additional variables to send to SqlRender::render +\subsection{\code{QueryNamespace$executeSql()}}{ + Call to execute sql within namespaced queries + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$executeSql(sql, ...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{sql}}{query string} + \item{\code{...}}{additional variables to send to SqlRender::render get vars} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-getVars}{}}} -\subsection{Method \code{getVars()}}{ -returns full list of variables that will be replaced +\subsection{\code{QueryNamespace$getVars()}}{ + returns full list of variables that will be replaced closeConnection -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$getVars()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$getVars()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-closeConnection}{}}} -\subsection{Method \code{closeConnection()}}{ -close connection, if active -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$closeConnection()}\if{html}{\out{
}} +\subsection{\code{QueryNamespace$closeConnection()}}{ + close connection, if active + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$closeConnection()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-QueryNamespace-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{QueryNamespace$clone(deep = FALSE)}\if{html}{\out{
}} +\subsection{\code{QueryNamespace$clone()}}{ + The objects of this class are cloneable with this method. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{QueryNamespace$clone(deep = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{deep}}{Whether to make a deep clone.} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} } diff --git a/man/ResultExportManager.Rd b/man/ResultExportManager.Rd index 29190de..72ddace 100644 --- a/man/ResultExportManager.Rd +++ b/man/ResultExportManager.Rd @@ -13,198 +13,202 @@ without issue. When exporting a the same table across multiple threads primary k issues. } \section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{exportDir}}{direcotry path to export files to + \if{html}{\out{
}} + \describe{ + \item{\code{exportDir}}{direcotry path to export files to Init} -} -\if{html}{\out{
}} + } + \if{html}{\out{
}} } \section{Methods}{ \subsection{Public methods}{ -\itemize{ -\item \href{#method-ResultExportManager-new}{\code{ResultExportManager$new()}} -\item \href{#method-ResultExportManager-getTableSpec}{\code{ResultExportManager$getTableSpec()}} -\item \href{#method-ResultExportManager-getMinColValues}{\code{ResultExportManager$getMinColValues()}} -\item \href{#method-ResultExportManager-checkRowTypes}{\code{ResultExportManager$checkRowTypes()}} -\item \href{#method-ResultExportManager-listTables}{\code{ResultExportManager$listTables()}} -\item \href{#method-ResultExportManager-checkPrimaryKeys}{\code{ResultExportManager$checkPrimaryKeys()}} -\item \href{#method-ResultExportManager-exportDataFrame}{\code{ResultExportManager$exportDataFrame()}} -\item \href{#method-ResultExportManager-exportQuery}{\code{ResultExportManager$exportQuery()}} -\item \href{#method-ResultExportManager-getManifestList}{\code{ResultExportManager$getManifestList()}} -\item \href{#method-ResultExportManager-writeManifest}{\code{ResultExportManager$writeManifest()}} -\item \href{#method-ResultExportManager-clone}{\code{ResultExportManager$clone()}} -} + \itemize{ + \item \href{#method-ResultExportManager-initialize}{\code{ResultExportManager$new()}} + \item \href{#method-ResultExportManager-getTableSpec}{\code{ResultExportManager$getTableSpec()}} + \item \href{#method-ResultExportManager-getMinColValues}{\code{ResultExportManager$getMinColValues()}} + \item \href{#method-ResultExportManager-checkRowTypes}{\code{ResultExportManager$checkRowTypes()}} + \item \href{#method-ResultExportManager-listTables}{\code{ResultExportManager$listTables()}} + \item \href{#method-ResultExportManager-checkPrimaryKeys}{\code{ResultExportManager$checkPrimaryKeys()}} + \item \href{#method-ResultExportManager-exportDataFrame}{\code{ResultExportManager$exportDataFrame()}} + \item \href{#method-ResultExportManager-exportQuery}{\code{ResultExportManager$exportQuery()}} + \item \href{#method-ResultExportManager-getManifestList}{\code{ResultExportManager$getManifestList()}} + \item \href{#method-ResultExportManager-writeManifest}{\code{ResultExportManager$writeManifest()}} + \item \href{#method-ResultExportManager-clone}{\code{ResultExportManager$clone()}} + } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ResultExportManager-new}{}}} -\subsection{Method \code{new()}}{ -Create a class for exporting results from a study in a standard, consistend manner -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$new( +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResultExportManager-initialize}{}}} +\subsection{\code{ResultExportManager$new()}}{ + Create a class for exporting results from a study in a standard, consistend manner + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$new( tableSpecification, exportDir, minCellCount = getOption("ohdsi.minCellCount", default = 5), validateTypes = FALSE, usePrimaryKeyCheck = FALSE, databaseId = NULL -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{tableSpecification}}{Table specification data.frame} - -\item{\code{exportDir}}{Directory files are being exported to} - -\item{\code{minCellCount}}{Minimum cell count - reccomended that you set with +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{tableSpecification}}{Table specification data.frame} + \item{\code{exportDir}}{Directory files are being exported to} + \item{\code{minCellCount}}{Minimum cell count - reccomended that you set with options("ohdsi.minCellCount" = count) in all R projects. Default is 5} - -\item{\code{validateTypes}}{Test if row values strictly conform to types - optional, not currently reccomended + \item{\code{validateTypes}}{Test if row values strictly conform to types - optional, not currently reccomended outside of development} - -\item{\code{usePrimaryKeyCheck}}{Test if primary key fields are violated at export step. - optional, not currently reccomended + \item{\code{usePrimaryKeyCheck}}{Test if primary key fields are violated at export step. - optional, not currently reccomended outside of development get table spec} - -\item{\code{databaseId}}{database identifier - required when exporting according to many specs} -} -\if{html}{\out{
}} -} + \item{\code{databaseId}}{database identifier - required when exporting according to many specs} + } + \if{html}{\out{
}} + } } + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-getTableSpec}{}}} -\subsection{Method \code{getTableSpec()}}{ -Get specification of table -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$getTableSpec(exportTableName)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{exportTableName}}{table name +\subsection{\code{ResultExportManager$getTableSpec()}}{ + Get specification of table + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$getTableSpec(exportTableName)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{exportTableName}}{table name Get min col values} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-getMinColValues}{}}} -\subsection{Method \code{getMinColValues()}}{ -Columns to convert to minimum for a given table name -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$getMinColValues(rows, exportTableName)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rows}}{data.frame of rows} - -\item{\code{exportTableName}}{stering table name - must be defined in spec +\subsection{\code{ResultExportManager$getMinColValues()}}{ + Columns to convert to minimum for a given table name + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$getMinColValues(rows, exportTableName)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{rows}}{data.frame of rows} + \item{\code{exportTableName}}{stering table name - must be defined in spec Check row types} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-checkRowTypes}{}}} -\subsection{Method \code{checkRowTypes()}}{ -Check types of rows before exporting -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$checkRowTypes(rows, exportTableName)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rows}}{data.frame of rows to export} - -\item{\code{exportTableName}}{table name +\subsection{\code{ResultExportManager$checkRowTypes()}}{ + Check types of rows before exporting + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$checkRowTypes(rows, exportTableName)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{rows}}{data.frame of rows to export} + \item{\code{exportTableName}}{table name List tables} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-listTables}{}}} -\subsection{Method \code{listTables()}}{ -list all tables in schema +\subsection{\code{ResultExportManager$listTables()}}{ + list all tables in schema Check primary keys of exported data -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$listTables()}\if{html}{\out{
}} + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$listTables()} + \if{html}{\out{
}} + } } -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-checkPrimaryKeys}{}}} -\subsection{Method \code{checkPrimaryKeys()}}{ -Checks to see if the rows conform to the valid primary keys +\subsection{\code{ResultExportManager$checkPrimaryKeys()}}{ + Checks to see if the rows conform to the valid primary keys If the same table has already been checked in the life of this object set "invalidateCache" to TRUE as the keys will be cached in a temporary file on disk. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$checkPrimaryKeys( + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$checkPrimaryKeys( rows, exportTableName, invalidateCache = FALSE -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rows}}{data.frame to export} - -\item{\code{exportTableName}}{Table name (must be in spec)} - -\item{\code{invalidateCache}}{logical - if starting a fresh export use this to delete cache of primary keys +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{rows}}{data.frame to export} + \item{\code{exportTableName}}{Table name (must be in spec)} + \item{\code{invalidateCache}}{logical - if starting a fresh export use this to delete cache of primary keys Export data frame} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-exportDataFrame}{}}} -\subsection{Method \code{exportDataFrame()}}{ -This method is intended for use where exporting a data.frame and not a query from a rdbms table +\subsection{\code{ResultExportManager$exportDataFrame()}}{ + This method is intended for use where exporting a data.frame and not a query from a rdbms table For example, if you perform a transformation in R this method will check primary keys, min cell counts and data types before writing the file to according to the table spec -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$exportDataFrame(rows, exportTableName, append = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rows}}{Rows to export} - -\item{\code{exportTableName}}{Table name} - -\item{\code{append}}{logical - if true will append the result to a file, otherwise the file will be overwritten + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$exportDataFrame(rows, exportTableName, append = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{rows}}{Rows to export} + \item{\code{exportTableName}}{Table name} + \item{\code{append}}{logical - if true will append the result to a file, otherwise the file will be overwritten Export Data table with sql query} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-exportQuery}{}}} -\subsection{Method \code{exportQuery()}}{ -Writes files in batch to stop overflowing system memory +\subsection{\code{ResultExportManager$exportQuery()}}{ + Writes files in batch to stop overflowing system memory Checks primary keys on write Checks minimum cell count -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$exportQuery( + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$exportQuery( connection, sql, exportTableName, @@ -212,97 +216,95 @@ Checks minimum cell count transformFunctionArgs = list(), append = FALSE, ... -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{connection}}{DatabaseConnector connection instance} - -\item{\code{sql}}{OHDSI sql string to export tables} - -\item{\code{exportTableName}}{Name of table to export (in snake_case format)} - -\item{\code{transformFunction}}{(optional) transformation of the data set callback. +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{connection}}{DatabaseConnector connection instance} + \item{\code{sql}}{OHDSI sql string to export tables} + \item{\code{exportTableName}}{Name of table to export (in snake_case format)} + \item{\code{transformFunction}}{(optional) transformation of the data set callback. must take two paramters - rows and pos \if{html}{\out{
}}\preformatted{ Following this transformation callback, results will be verified against data model, Primary keys will be checked and minCellValue rules will be enforced }\if{html}{\out{
}}} - -\item{\code{transformFunctionArgs}}{arguments to be passed to the transformation function} - -\item{\code{append}}{Logical add results to existing file, if FALSE (default) creates a new file and removes primary + \item{\code{transformFunctionArgs}}{arguments to be passed to the transformation function} + \item{\code{append}}{Logical add results to existing file, if FALSE (default) creates a new file and removes primary key validation cache} - -\item{\code{...}}{extra parameters passed to sql + \item{\code{...}}{extra parameters passed to sql get manifest list} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-getManifestList}{}}} -\subsection{Method \code{getManifestList()}}{ -Create a meta data set for each collection of result files with sha256 has for all files -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$getManifestList( +\subsection{\code{ResultExportManager$getManifestList()}}{ + Create a meta data set for each collection of result files with sha256 has for all files + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$getManifestList( packageName = NULL, packageVersion = NULL, migrationsPath = NULL, migrationRegexp = .defaultMigrationRegexp -)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{packageName}}{if an R analysis package, specify the name} - -\item{\code{packageVersion}}{if an analysis package, specify the version} - -\item{\code{migrationsPath}}{path to sql migrations (use top level folder (e.g. sql/sql_server/migrations)} - -\item{\code{migrationRegexp}}{(optional) regular expression to search for sql files. It is not reccomended to change the default. +)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{packageName}}{if an R analysis package, specify the name} + \item{\code{packageVersion}}{if an analysis package, specify the version} + \item{\code{migrationsPath}}{path to sql migrations (use top level folder (e.g. sql/sql_server/migrations)} + \item{\code{migrationRegexp}}{(optional) regular expression to search for sql files. It is not reccomended to change the default. Write manifest} + } + \if{html}{\out{
}} + } } -\if{html}{\out{
}} -} -} + \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-writeManifest}{}}} -\subsection{Method \code{writeManifest()}}{ -Write manifest json -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$writeManifest(...)}\if{html}{\out{
}} +\subsection{\code{ResultExportManager$writeManifest()}}{ + Write manifest json + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$writeManifest(...)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{...}}{@seealso getManifestList} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{...}}{@seealso getManifestList} -} -\if{html}{\out{
}} -} -} \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResultExportManager-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultExportManager$clone(deep = FALSE)}\if{html}{\out{
}} +\subsection{\code{ResultExportManager$clone()}}{ + The objects of this class are cloneable with this method. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{ResultExportManager$clone(deep = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{deep}}{Whether to make a deep clone.} + } + \if{html}{\out{
}} + } } -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} } diff --git a/man/ResultModelManager-package.Rd b/man/ResultModelManager-package.Rd index e8c65b5..16b4211 100644 --- a/man/ResultModelManager-package.Rd +++ b/man/ResultModelManager-package.Rd @@ -20,5 +20,10 @@ Useful links: \author{ \strong{Maintainer}: Jamie Gilbert \email{gilbert@ohdsi.org} +Authors: +\itemize{ + \item Jamie Gilbert \email{gilbert@ohdsi.org} +} + } \keyword{internal} diff --git a/man/csvToYaml.Rd b/man/csvToYaml.Rd new file mode 100644 index 0000000..3f99b84 --- /dev/null +++ b/man/csvToYaml.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataModel.R +\name{csvToYaml} +\alias{csvToYaml} +\title{Convert a CSV specification to YAML format} +\usage{ +csvToYaml( + csvFilepath, + yamlOutputPath, + namespace = "default", + overwrite = FALSE +) +} +\arguments{ +\item{csvFilepath}{Path to the CSV specification file} + +\item{yamlOutputPath}{Path to write the YAML output file} + +\item{namespace}{Namespace name to use for all tables in the output. +Defaults to "default".} + +\item{overwrite}{Boolean - overwrite existing output file?} +} +\value{ +Invisibly returns the YAML content as a string +} +\description{ +Reads a CSV results data model specification file and writes the equivalent +YAML file. This is a migration helper to transition from the legacy CSV format +to the namespaced YAML format. +} diff --git a/man/generateSqlSchema.Rd b/man/generateSqlSchema.Rd index 015384a..0354c64 100644 --- a/man/generateSqlSchema.Rd +++ b/man/generateSqlSchema.Rd @@ -8,21 +8,30 @@ generateSqlSchema( csvFilepath = NULL, schemaDefinition = NULL, sqlOutputPath = NULL, - overwrite = FALSE + overwrite = FALSE, + platform = NULL ) } \arguments{ -\item{csvFilepath}{Path to schema file. Csv file must have the columns: -"table_name", "column_name", "data_type", "primary_key"} +\item{csvFilepath}{Path to schema file (csv or yaml). Csv file must have the columns: +"table_name", "column_name", "data_type", "primary_key". +Yaml file must follow the namespaced YAML schema format.} -\item{schemaDefinition}{A schemaDefintiion data.frame` with the columns: -tableName, columnName, dataType, isRequired, primaryKey} +\item{schemaDefinition}{A schemaDefinition data.frame with the columns: +tableName, columnName, dataType, isRequired, primaryKey. +May optionally include a 'namespace' column.} \item{sqlOutputPath}{File to write sql to.} \item{overwrite}{Boolean - overwrite existing file?} + +\item{platform}{Target database platform for platform-specific DDL +(e.g. "postgresql", "sql_server", "sqlite", "duckdb"). +Only applies when loading from a YAML file.} } \description{ -Take a csv schema definition and create a basic sql script with it. +Take a csv or yaml schema definition and create a basic sql script with it. +For YAML files with platform-specific configuration, use the \code{platform} parameter +to include features like partitioning and indexes. returns string containing the sql for the table } diff --git a/man/loadResultsDataModelFromYaml.Rd b/man/loadResultsDataModelFromYaml.Rd new file mode 100644 index 0000000..6f73684 --- /dev/null +++ b/man/loadResultsDataModelFromYaml.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/DataModel.R +\name{loadResultsDataModelFromYaml} +\alias{loadResultsDataModelFromYaml} +\title{Load results data model from YAML file} +\usage{ +loadResultsDataModelFromYaml(filePath) +} +\arguments{ +\item{filePath}{Path to a valid YAML file} +} +\value{ +A list with elements: +\item{specification}{A tibble data frame with columns: namespace, namespacePrefix, +tableName, tableDescription, columnName, dataType, primaryKey, nullable, +isRequired, optional, minCellCount, emptyIsNa, references, description} +\item{platforms}{Platform-specific configuration (postgresql, sql_server, duckdb, sqlite) or NULL} +} +\description{ +Load a YAML results data model specification file. Returns a list containing +the flattened specification data frame and optional platform-specific configuration. + +Each HADES package should ship its own \code{resultsDataModelSpecification.yaml} in +its \code{inst/settings/} directory. A package's YAML defines only the tables +produced by that package. + +Combining multiple YAML specifications (e.g. from different packages) into a +unified HADES results schema is the responsibility of a higher-level +orchestrator such as Strategus or a future \code{OhdsiResultsManager} package. +This package provides per-spec loading only, to avoid circular dependencies +between packages that share no direct relationship at install time. +} diff --git a/man/loadResultsDataModelSpecifications.Rd b/man/loadResultsDataModelSpecifications.Rd index 76fccaa..551174d 100644 --- a/man/loadResultsDataModelSpecifications.Rd +++ b/man/loadResultsDataModelSpecifications.Rd @@ -7,7 +7,7 @@ loadResultsDataModelSpecifications(filePath) } \arguments{ -\item{filePath}{path to a valid csv file} +\item{filePath}{path to a valid csv or yaml file} } \value{ A tibble data frame object with specifications From 1aed90a43f0ee4b1f0be9323c9349a03016e357e Mon Sep 17 00:00:00 2001 From: Jamie Gilbert Date: Wed, 15 Jul 2026 17:32:34 -0700 Subject: [PATCH 8/8] attempt to fix annoying github actions issue --- R/QueryNamespace.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/R/QueryNamespace.R b/R/QueryNamespace.R index 949ca35..b5eacc1 100644 --- a/R/QueryNamespace.R +++ b/R/QueryNamespace.R @@ -290,7 +290,10 @@ QueryNamespace <- R6::R6Class( params$sql <- sql params$warnOnMissingParameters <- FALSE - do.call(SqlRender::render, params) + utils::capture.output({ + result <- do.call(SqlRender::render, params) + }) + return(result) }, #' query Sql