diff --git a/DESCRIPTION b/DESCRIPTION index b3ff708..64266d6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -53,5 +53,6 @@ biocViews: GenomeAssembly, Annotation, Sequencing VignetteBuilder: knitr Suggests: rmarkdown, + writexl, testthat (>= 3.0.0) Config/roxygen2/version: 8.0.0 diff --git a/NAMESPACE b/NAMESPACE index 69af5ae..70f9411 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,8 @@ export(CDHIT2duckdb) export(buildClusterFeatureMap) export(cleanData) export(cleanMetaData) +export(exportProcessedData) +export(exportTables) export(generatePlots) export(generateSummary) export(prepareGenomes) diff --git a/R/data_curation.R b/R/data_curation.R index 7edc518..b6b803f 100644 --- a/R/data_curation.R +++ b/R/data_curation.R @@ -5,6 +5,30 @@ grepl("^[0-9]+$", x) } +#' A helper used in data_curation.R and data_processing.R to ensure exported tables +#' don't lose trailing zeroes. Should be relocated into a common helpers/utilities +#' script later. +#' @keywords internal +.preserve_export_id_text <- function(df) { + df <- tibble::as_tibble(df) + + id_pattern <- paste0( + "(^|[._])(", + "genome(_drug)?_id|taxon_id|", + "assembly_accession|bioproject_accession|biosample_accession|", + "refseq_accessions?|genbank_accessions?|sra_accession|pmid|", + "gene_id|protein_id|domain_id|cluster_id|AccNum|id", + ")$" + ) + + id_cols <- names(df)[grepl(id_pattern, names(df), ignore.case = TRUE)] + if (length(id_cols)) { + df[id_cols] <- lapply(df[id_cols], as.character) + } + + df +} + #' Helps tag genomes with their AMR evidence for parsing #' @keywords internal .create_amr_tagged_view <- function(con) { @@ -288,8 +312,8 @@ #' Deviation-based filters are optional and are best suited to single-taxon runs. #' #' @param genome_tbl A tibble of genome metadata containing BV-BRC genome columns. -#' @param checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). -#' @param checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). +#' @param max_checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). +#' @param min_checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). #' @param gc_deviations Optional numeric scalar. Maximum SDs from the median GC content. #' @param length_deviations Optional numeric scalar. Maximum SDs from the median genome length. #' @param cds_deviations Optional numeric scalar. Maximum SDs from the median CDS count. @@ -302,8 +326,8 @@ #' } #' @keywords internal .apply_metadata_qc <- function(genome_tbl, - checkm_contam = 5, - checkm_complete = 95, + max_checkm_contam = 5, + min_checkm_complete = 95, gc_deviations = NULL, length_deviations = NULL, cds_deviations = NULL) { @@ -362,8 +386,8 @@ qc <- qc |> dplyr::mutate( flag_checkm_missing = is.na(.data$checkm_completeness_qc) | is.na(.data$checkm_contamination_qc), - flag_checkm_contam = !is.na(.data$checkm_contamination_qc) & .data$checkm_contamination_qc > checkm_contam, - flag_checkm_complete = !is.na(.data$checkm_completeness_qc) & .data$checkm_completeness_qc < checkm_complete, + flag_checkm_contam = !is.na(.data$checkm_contamination_qc) & .data$checkm_contamination_qc > max_checkm_contam, + flag_checkm_complete = !is.na(.data$checkm_completeness_qc) & .data$checkm_completeness_qc < min_checkm_complete, flag_checkm = .data$flag_checkm_missing | .data$flag_checkm_contam | .data$flag_checkm_complete, flag_length = if (!is.null(length_deviations) && is.finite(sd_len) && sd_len > 0) { !is.na(z_len) & z_len > length_deviations @@ -383,7 +407,7 @@ qc_keep = !(.data$flag_checkm | .data$flag_length | .data$flag_gc | .data$flag_cds) ) - make_rej <- function(flag, rule, observed, threshold, comparator) { + make_rejects <- function(flag, rule, observed, threshold, comparator) { idx <- which(flag %in% TRUE) if (!length(idx)) { return(NULL) @@ -400,32 +424,32 @@ ) } - rej_list <- list( - make_rej( + reject_list <- list( + make_rejects( qc$flag_checkm_missing, "checkm_missing", ifelse(qc$flag_checkm_missing, "missing", NA_character_), "present", "is present" ), - make_rej( + make_rejects( qc$flag_checkm_contam, "checkm_contamination", qc$checkm_contamination_qc, - checkm_contam, + max_checkm_contam, ">" ), - make_rej( + make_rejects( qc$flag_checkm_complete, "checkm_completeness", qc$checkm_completeness_qc, - checkm_complete, + min_checkm_complete, "<" ) ) if (!is.null(gc_deviations)) { - rej_list[[length(rej_list) + 1L]] <- make_rej( + reject_list[[length(reject_list) + 1L]] <- make_rejects( qc$flag_gc, "gc_deviation_sd", z_gc, @@ -435,7 +459,7 @@ } if (!is.null(length_deviations)) { - rej_list[[length(rej_list) + 1L]] <- make_rej( + reject_list[[length(reject_list) + 1L]] <- make_rejects( qc$flag_length, "length_deviation_sd", z_len, @@ -445,7 +469,7 @@ } if (!is.null(cds_deviations)) { - rej_list[[length(rej_list) + 1L]] <- make_rej( + reject_list[[length(reject_list) + 1L]] <- make_rejects( qc$flag_cds, "cds_deviation_sd", z_cds, @@ -454,7 +478,7 @@ ) } - rejections <- dplyr::bind_rows(rej_list) + rejections <- dplyr::bind_rows(reject_list) if (nrow(rejections) == 0L) { rejections <- tibble::tibble( genome_id = character(), @@ -1191,8 +1215,8 @@ #' @param abx Character or vector. Antibiotic filter. "All" for all antibiotics, else names. #' @param overwrite Logical. If FALSE and DuckDB exists already, abort. Default FALSE. #' @param image Character. Docker image. Default "danylmb/bvbrc:5.3". -#' @param checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). -#' @param checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). +#' @param max_checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). +#' @param min_checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). #' @param gc_deviations Optional numeric scalar. Maximum SDs from the median GC content. #' @param length_deviations Optional numeric scalar. Maximum SDs from the median genome length. #' @param cds_deviations Optional numeric scalar. Maximum SDs from the median CDS count. @@ -1210,12 +1234,14 @@ retrieveMetadata <- function(user_bacs, abx = "All", overwrite = FALSE, image = "danylmb/bvbrc:5.3", - checkm_contam = 5, - checkm_complete = 95, + max_checkm_contam = 5, + min_checkm_complete = 95, gc_deviations = NULL, length_deviations = NULL, cds_deviations = NULL, debug = FALSE, + export_tables = FALSE, + load_tables = FALSE, verbose = TRUE) { base_dir <- normalizePath(base_dir, mustWork = FALSE) @@ -1390,8 +1416,8 @@ retrieveMetadata <- function(user_bacs, qc_out <- .apply_metadata_qc( genome_tbl = combined_genome_data_tbl, - checkm_contam = checkm_contam, - checkm_complete = checkm_complete, + max_checkm_contam = max_checkm_contam, + min_checkm_complete = min_checkm_complete, gc_deviations = gc_deviations, length_deviations = length_deviations, cds_deviations = cds_deviations @@ -1519,6 +1545,24 @@ retrieveMetadata <- function(user_bacs, } } + export_res <- NULL + if (isTRUE(export_tables) || isTRUE(load_tables)) { + export_res <- exportTables( + paths$db_path, + export_tables = export_tables, + load_tables = load_tables, + verbose = verbose + ) + } + + if (isTRUE(load_tables)) { + return(list( + duckdbConnection = con, + table_name = "metadata", + data = if (!is.null(export_res)) export_res$data else NULL + )) + } + list(duckdbConnection = con, table_name = "metadata") } @@ -1943,7 +1987,6 @@ retrieveGenomes <- function(base_dir = ".", cli_gff_workers = 8L, chunk_size = 50L, evidence_mode = c("lab_only", "lab_or_comp", "comp_only", "any"), - # NEW verbose = TRUE) { method <- match.arg(method) evidence_mode <- match.arg(evidence_mode) @@ -1951,7 +1994,7 @@ retrieveGenomes <- function(base_dir = ".", # Use 'filtered' if already prepared, or start filtering if (isTRUE(verbose)) - message("Preparing download set (checking for existing 'filtered').") + message("Preparing download set (checking for existing 'filtered' table).") paths <- .buildDBpath(base_dir = base_dir, user_bacs = user_bacs) db_path <- paths$db_path con0 <- DBI::dbConnect(duckdb::duckdb(), dbdir = db_path) @@ -1959,7 +2002,7 @@ retrieveGenomes <- function(base_dir = ".", if (has_filtered) { if (isTRUE(verbose)) - message("Using existing 'filtered' table (skipping re-filter).") + message("Found existing 'filtered' table.") con <- con0 tbl <- "filtered" on.exit(try(DBI::dbDisconnect(con0, shutdown = TRUE), silent = TRUE), add = TRUE) @@ -1979,6 +2022,7 @@ retrieveGenomes <- function(base_dir = ".", on.exit(try(DBI::dbDisconnect(con, shutdown = TRUE), silent = TRUE), add = TRUE) } + # What genomes need to be downloaded? Build set as `ids` ids <- tibble::as_tibble(DBI::dbReadTable(con, tbl)) |> dplyr::distinct(`genome.genome_id`) |> dplyr::pull(`genome.genome_id`) @@ -1989,6 +2033,7 @@ retrieveGenomes <- function(base_dir = ".", dir.create(genome_path, recursive = TRUE, showWarnings = FALSE) dir.create(logs_dir, recursive = TRUE, showWarnings = FALSE) + # Checks what is already downloaded vs. the full list needed; takes diff if (isTRUE(skip_existing)) { already <- .list_complete(genome_path, ids) if (isTRUE(verbose)) @@ -1996,9 +2041,10 @@ retrieveGenomes <- function(base_dir = ".", ids <- setdiff(ids, already) } + # Is diff length 0? If so, all genomes ready to go! if (length(ids) == 0L) { if (isTRUE(verbose)) - message("All genomes already complete.") + message("Download of all genomes already complete.") all_ids <- tibble::as_tibble(DBI::dbReadTable(con, tbl)) |> dplyr::distinct(`genome.genome_id`) |> dplyr::pull(`genome.genome_id`) @@ -2207,8 +2253,8 @@ genomeList <- function(base_dir = ".", #' return very large download lists for many species! #' @param num_workers Integer. Parallel workers used for genome download. #' Applied to both FTP and CLI download branches. Default: 8. -#' @param checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). -#' @param checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). +#' @param max_checkm_contam Numeric scalar. Maximum allowed CheckM contamination (%). +#' @param min_checkm_complete Numeric scalar. Minimum allowed CheckM completeness (%). #' @param gc_deviations Optional numeric scalar. Maximum SDs from the median GC content. #' @param length_deviations Optional numeric scalar. Maximum SDs from the median genome length. #' @param cds_deviations Optional numeric scalar. Maximum SDs from the median CDS count. @@ -2227,11 +2273,13 @@ prepareGenomes <- function(user_bacs, overwrite = FALSE, num_workers = 8L, evidence_mode = c("lab_only", "lab_or_comp", "comp_only", "any"), - checkm_contam = 5, - checkm_complete = 95, + max_checkm_contam = 5, + min_checkm_complete = 95, gc_deviations = NULL, length_deviations = NULL, cds_deviations = NULL, + export_tables = FALSE, + load_tables = FALSE, debug = FALSE, verbose = TRUE) { method <- match.arg(method) @@ -2248,8 +2296,8 @@ prepareGenomes <- function(user_bacs, base_dir = base_dir, abx = "All", overwrite = overwrite, - checkm_contam = checkm_contam, - checkm_complete = checkm_complete, + max_checkm_contam = max_checkm_contam, + min_checkm_complete = min_checkm_complete, gc_deviations = gc_deviations, length_deviations = length_deviations, cds_deviations = cds_deviations, @@ -2315,5 +2363,257 @@ prepareGenomes <- function(user_bacs, message("Continue with downstream processing using:") message('runDataProcessing("', normalizePath(paths$db_path), '")') } + + export_res <- NULL + if (isTRUE(export_tables) || isTRUE(load_tables)) { + export_res <- exportTables( + paths$db_path, + export_tables = export_tables, + load_tables = load_tables, + verbose = verbose + ) + } + + if (isTRUE(load_tables)) { + return(list( + duckdb_path = paths$db_path, + table_name = "files", + data = if (!is.null(export_res)) export_res$data else NULL + )) + } + invisible(out) } + +#' Export DuckDB tables and optionally load them into R +#' +#' Writes selected DuckDB tables to CSV files and optionally returns them as +#' in-memory R data frames. +#' +#' @param duckdb_path Character. Path to the DuckDB file. +#' @param output_dir Character or NULL. Directory for exports. Defaults to +#' file.path(dirname(duckdb_path), "exports"). +#' @param tables Character vector or NULL. Tables to export. If NULL, exports +#' all tables in the database. +#' @param skip_tables Character vector of table names to exclude. +#' @param include_summary Logical. If TRUE, writes summary.csv and summary.txt. +#' @param load_tables Logical. If TRUE, also return the exported tables as +#' in-memory R data frames. +#' @param verbose Logical. If TRUE, prints progress messages. +#' +#' @return Invisibly returns a list with exported file paths and, if requested, +#' in-memory tables. +#' @export +exportTables <- function(duckdb_path, + output_dir = NULL, + tables = NULL, + skip_tables = NULL, + include_summary = TRUE, + export_formats = c("csv"), + export_tables = TRUE, + load_tables = FALSE, + verbose = TRUE) { + duckdb_path <- normalizePath(duckdb_path, mustWork = TRUE) + + if (is.null(output_dir)) { + output_dir <- file.path(dirname(duckdb_path), "exports") + } + output_dir <- normalizePath(output_dir, mustWork = FALSE) + dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) + + export_formats <- unique(tolower(export_formats)) + export_formats[export_formats == "excel"] <- "xlsx" + + allowed_formats <- c("csv", "tsv", "xlsx", "parquet") + unknown_formats <- setdiff(export_formats, allowed_formats) + if (length(unknown_formats)) { + stop("Unsupported export format(s): ", paste(unknown_formats, collapse = ", ")) + } + + if (isTRUE(export_tables) && !length(export_formats)) { + stop("At least one export format must be supplied when export_tables = TRUE.") + } + + warn_text_exports <- any(export_formats %in% c("csv", "tsv", "xlsx")) + if (isTRUE(export_tables) && warn_text_exports) { + message( + "\nNote: CSV, TSV, and Excel exports are intended primarily for human readability.\n", + "However, BV-BRC genome accession are differentiated by trailing zero values.\n", + "Example: 1282.2280 is a different genome than 1282.228\n", + "If reading these files into software like Excel, or even re-reading them into R,\n", + "accession IDs can be read as 'numeric' and trailing zeroes dropped!\n", + "For programmatic reuse, we suggest using Parquet format, or \n", + "explicitly import accession ID columns as 'character', not 'numeric'.\n" + ) + } + + if ("xlsx" %in% export_formats && !requireNamespace("writexl", quietly = TRUE)) { + stop("Format 'xlsx' was requested but package 'writexl' is not available.") + } + + con <- DBI::dbConnect(duckdb::duckdb(), dbdir = duckdb_path) + on.exit(try(DBI::dbDisconnect(con, shutdown = TRUE), silent = TRUE), add = TRUE) + + available_tables <- DBI::dbListTables(con) + if (!length(available_tables)) { + stop("No tables found in DuckDB: ", duckdb_path) + } + + default_tables <- c( + "bac_data", + "filtered", + "metadata", + "genome_data", + "amr_phenotype", + "metadata_qc", + "metadata_qc_rejections", + "files" + ) + + selected_tables <- if (is.null(tables)) { + intersect(default_tables, available_tables) + } else { + intersect(unique(as.character(tables)), available_tables) + } + + if (!is.null(skip_tables) && length(skip_tables)) { + selected_tables <- setdiff(selected_tables, skip_tables) + } + + if (!length(selected_tables)) { + stop("No tables left to export after applying skip_tables.") + } + + preserve_id_text <- .preserve_export_id_text + + write_one <- function(df, stem, fmt) { + df <- preserve_id_text(df) + out_file <- file.path(output_dir, paste0(stem, ".", fmt)) + + if (fmt == "csv") { + utils::write.table( + df, + file = out_file, + sep = ",", + row.names = FALSE, + col.names = TRUE, + quote = TRUE, + na = "", + qmethod = "double", + fileEncoding = "UTF-8" + ) + } else if (fmt == "tsv") { + utils::write.table( + df, + file = out_file, + sep = "\t", + row.names = FALSE, + col.names = TRUE, + quote = TRUE, + na = "", + qmethod = "double", + fileEncoding = "UTF-8" + ) + } else if (fmt == "xlsx") { + writexl::write_xlsx(list(data = df), out_file) + } else if (fmt == "parquet") { + arrow::write_parquet(df, out_file) + } else { + stop("Unhandled format: ", fmt) + } + + out_file + } + + exported_files <- list() + loaded_tables <- list() + + for (tbl in selected_tables) { + df <- tibble::as_tibble(DBI::dbReadTable(con, tbl)) + + if (isTRUE(export_tables)) { + for (fmt in export_formats) { + out_file <- write_one(df, tbl, fmt) + exported_files[[paste(tbl, fmt, sep = ".")]] <- out_file + } + if (isTRUE(verbose)) { + message("Exported table: ", tbl, " -> ", paste(export_formats, collapse = ", ")) + } + } + + if (isTRUE(load_tables)) { + loaded_tables[[tbl]] <- preserve_id_text(df) + } + } + + count_if_present <- function(tbl) { + if (tbl %in% available_tables) { + as.character(DBI::dbGetQuery( + con, + paste0("SELECT COUNT(*) AS n FROM ", DBI::dbQuoteIdentifier(con, tbl)) + )$n[[1]]) + } else { + NA_character_ + } + } + + summary_tables <- available_tables + summary_tbl <- tibble::tibble( + metric = c( + "duckdb_path", + "export_dir", + "export_formats", + "tables_requested", + "tables_exported", + "table_names", + paste0(summary_tables, "_rows") + ), + value = c( + duckdb_path, + output_dir, + paste(export_formats, collapse = ", "), + if (is.null(tables)) "default" else paste(unique(as.character(tables)), collapse = ", "), + as.character(length(exported_files)), + paste(summary_tables, collapse = ", "), + vapply(summary_tables, count_if_present, character(1)) + ) + ) + + if (isTRUE(export_tables) && isTRUE(include_summary)) { + utils::write.table( + summary_tbl, + file = file.path(output_dir, "summary.csv"), + sep = ",", + row.names = FALSE, + col.names = TRUE, + quote = TRUE, + na = "", + qmethod = "double", + fileEncoding = "UTF-8" + ) + writeLines( + c( + paste0("DuckDB: ", duckdb_path), + paste0("Export directory: ", output_dir), + paste0("Export formats: ", paste(export_formats, collapse = ", ")), + paste0("Tables exported: ", length(exported_files)), + paste0("Table names: ", paste(summary_tables, collapse = ", ")), + "", + "Caution: CSV/TSV/XLSX exports may be re-read as numeric by default. Force accession/ID columns to character on import. Parquet is safer for programmatic reuse." + ), + file.path(output_dir, "summary.txt"), + useBytes = TRUE + ) + if (isTRUE(verbose)) { + message("Exported summary files.") + } + } + + invisible(list( + export_dir = output_dir, + tables = selected_tables, + files = exported_files, + data = if (isTRUE(load_tables)) loaded_tables else NULL, + summary = summary_tbl + )) +} diff --git a/R/data_processing.R b/R/data_processing.R index 945f59d..a0cfbf9 100644 --- a/R/data_processing.R +++ b/R/data_processing.R @@ -1337,10 +1337,10 @@ cleanMetaData <- function(duckdb_path, path, ref_file_path = "data_raw/") { dplyr::select("raw_entry", "clean_name", "short_name") |> dplyr::distinct() - # Define lab methods + # Define lab methods lab_methods <- c("Disk diffusion", "MIC", "Broth dilution", "Agar dilution", "Biofosun Gram-positive panels broth dilution", "Vitek_2-P607_card", "cation-adjusted Mueller-Hinton broth", "gradient_diffusion", "kirby-bauer_disc_diffusion") - + dplyr::tbl(con, "filtered") |> tibble::as_tibble() |> dplyr::select("genome.genome_id") |> @@ -1355,9 +1355,9 @@ cleanMetaData <- function(duckdb_path, path, ref_file_path = "data_raw/") { "genome.isolation_source", "genome.species" ) |> dplyr::mutate(genome_drug.evidence = dplyr::case_when( - genome_drug.laboratory_typing_method %in% lab_methods ~ "Laboratory Method", + genome_drug.laboratory_typing_method %in% lab_methods ~ "Laboratory Method", genome_drug.laboratory_typing_method == "Computational Prediction" ~ "Computational Method", - TRUE ~ genome_drug.evidence)) |> + TRUE ~ genome_drug.evidence)) |> dplyr::filter(genome_drug.evidence == "Laboratory Method") |> dplyr::left_join(clean_drug, by = c("genome_drug.antibiotic" = "original_drug")) |> dplyr::filter(!is.na(cleaned_drug)) |> @@ -1822,3 +1822,249 @@ runDataProcessing <- function(duckdb_path, parquet_duckdb_path = normalizePath(parquet_duckdb_path) )) } + +#' Export processed tables from DuckDB database +#' +#' Reads tables from the DuckDB database produced by the `runDataProcessing()` workflow +#' and exports them as CSV, TSV, Parquet, and/or XLSX. This is an optional step +#' that allows users to take their processed data outside our amR workflow for +#' use in their own custom analyses. This is not required to run `amRml`! +#' +#' @param duckdb_path Character. Path to the DuckDB database created by the +#' workflow (for example, `Sar.duckdb`). +#' @param output_path Character or NULL. Directory for exports. Defaults to +#' file.path(dirname(duckdb_path), "processed_exports"). +#' @param amr_phenotype_mode Character. One of "separate" or "append". +#' "separate" exports the AMR labels as a separate wide table. +#' "append" joins those labels onto the main feature tables before export. +#' @param export_formats Character vector. Any of "csv", "tsv", "parquet", "xlsx". +#' @param tables Character vector or NULL. Tables to export. If NULL, exports the +#' standard processed tables present in the database. +#' @param verbose Logical. If TRUE, prints progress messages. +#' +#' @return Invisibly returns a list containing the export path, table names, and mode. +#' @export +exportProcessedData <- function(duckdb_path, + output_path = NULL, + amr_phenotype_mode = c("separate", "append"), + export_formats = c("csv"), + export_sequences = FALSE, + tables = NULL, + export_tables = TRUE, + verbose = TRUE) { + duckdb_path <- normalizePath(duckdb_path, mustWork = TRUE) + amr_phenotype_mode <- match.arg(amr_phenotype_mode) + + export_formats <- unique(tolower(export_formats)) + export_formats[export_formats == "excel"] <- "xlsx" + + allowed_formats <- c("csv", "tsv", "xlsx", "parquet") + unknown_formats <- setdiff(export_formats, allowed_formats) + if (length(unknown_formats)) { + stop("Unsupported export format(s): ", paste(unknown_formats, collapse = ", ")) + } + + if (isTRUE(export_tables) && !length(export_formats)) { + stop("At least one export format must be supplied when export_tables = TRUE.") + } + + warn_text_exports <- any(export_formats %in% c("csv", "tsv", "xlsx")) + if (isTRUE(export_tables) && warn_text_exports) { + message( + "\nNote: CSV, TSV, and Excel exports are intended primarily for human readability.\n", + "However, BV-BRC genome accession are differentiated by trailing zero values.\n", + "Example: 1282.2280 is a different genome than 1282.228\n", + "If reading these files into software like Excel, or even re-reading them into R,\n", + "accession IDs can be read as 'numeric' and trailing zeroes dropped!\n", + "For programmatic reuse, we suggest using Parquet format, or \n", + "explicitly import accession ID columns as 'character', not 'numeric'.\n" + ) + } + + if ("xlsx" %in% export_formats && !requireNamespace("writexl", quietly = TRUE)) { + stop("Format 'xlsx' was requested but package 'writexl' is not available.") + } + + if (is.null(output_path)) { + output_path <- file.path(dirname(duckdb_path), "processed_exports") + } + output_path <- normalizePath(output_path, mustWork = FALSE) + dir.create(output_path, recursive = TRUE, showWarnings = FALSE) + + con <- DBI::dbConnect(duckdb::duckdb(), dbdir = duckdb_path, read_only = TRUE) + DBI::dbExecute( + con, + sprintf( + "SET file_search_path='%s'", + dirname(normalizePath(duckdb_path)) + ) + ) + on.exit(try(DBI::dbDisconnect(con, shutdown = TRUE), silent = TRUE), add = TRUE) + + available_tables <- DBI::dbListTables(con) + if (!length(available_tables)) { + stop("No tables found in DuckDB: ", duckdb_path) + } + + read_tbl <- function(tbl) { + tibble::as_tibble(DBI::dbReadTable(con, tbl)) + } + + write_one <- function(df, stem) { + df <- .preserve_export_id_text(df) + + if ("csv" %in% export_formats) { + utils::write.table( + df, file = file.path(output_path, paste0(stem, ".csv")), + sep = ",", row.names = FALSE, col.names = TRUE, + quote = TRUE, na = "", qmethod = "double", fileEncoding = "UTF-8" + ) + } + if ("tsv" %in% export_formats) { + utils::write.table( + df, file = file.path(output_path, paste0(stem, ".tsv")), + sep = "\t", row.names = FALSE, col.names = TRUE, + quote = TRUE, na = "", qmethod = "double", fileEncoding = "UTF-8" + ) + } + if ("parquet" %in% export_formats) { + arrow::write_parquet(df, file.path(output_path, paste0(stem, ".parquet"))) + } + if ("xlsx" %in% export_formats) { + writexl::write_xlsx(list(data = df), file.path(output_path, paste0(stem, ".xlsx"))) + } + } + + build_amr_wide <- function() { + source_tbl <- if ("metadata" %in% available_tables) { + "metadata" + } else if ("amr_phenotype" %in% available_tables) { + "amr_phenotype" + } else { + NULL + } + + if (is.null(source_tbl)) { + return(NULL) + } + + md <- read_tbl(source_tbl) + + needed <- c("genome.genome_id", "genome_drug.antibiotic", "genome_drug.resistant_phenotype") + if (!all(needed %in% names(md))) { + return(NULL) + } + + md |> + dplyr::transmute( + genome_id = as.character(`genome.genome_id`), + antibiotic = as.character(`genome_drug.antibiotic`), + phenotype = as.character(`genome_drug.resistant_phenotype`) + ) |> + dplyr::filter(!is.na(genome_id), !is.na(antibiotic), !is.na(phenotype)) |> + dplyr::distinct() |> + dplyr::group_by(genome_id, antibiotic) |> + dplyr::summarise( + phenotype = paste(sort(unique(phenotype)), collapse = ";"), + .groups = "drop" + ) |> + tidyr::pivot_wider( + names_from = antibiotic, + values_from = phenotype, + values_fill = NA_character_ + ) |> + dplyr::arrange(genome_id) + } + + table_specs <- list( + gene_count = list(source = "gene_count", stem = "gene_count", appendable = TRUE), + protein_count = list(source = "protein_count", stem = "protein_count", appendable = TRUE), + domain_count = list(source = "domain_count", stem = "domain_count", appendable = TRUE), + struct = list(source = "struct", stem = "struct", appendable = TRUE), + gene_names = list(source = "gene_names", stem = "gene_names", appendable = FALSE), + protein_names = list(source = "protein_names", stem = "protein_names", appendable = FALSE), + domain_names = list(source = "domain_names", stem = "domain_names", appendable = FALSE), + metadata = list(source = "metadata", stem = "metadata", appendable = FALSE), + genome_data = list(source = "genome_data", stem = "genome_data", appendable = FALSE), + amr_phenotype_wide = list(source = NULL, stem = "amr_phenotype_wide", appendable = FALSE) + ) + + if (isTRUE(export_sequences)) { + table_specs$gene_seqs <- list(source = "gene_ref_seq", stem = "gene_seqs", appendable = FALSE) + table_specs$protein_seqs <- list(source = "protein_cluster_seq", stem = "protein_seqs", appendable = FALSE) + table_specs$genome_gene_protein <- list(source = "genome_gene_protein", stem = "genome_gene_protein", appendable = FALSE) + } + + if (is.null(tables)) { + selected_keys <- c( + "gene_count", + "protein_count", + "domain_count", + "struct", + "gene_names", + "protein_names", + "domain_names", + "metadata", + "genome_data", + "amr_phenotype_wide" + ) + if (isTRUE(export_sequences)) { + selected_keys <- c(selected_keys, "gene_seqs", "protein_seqs", "genome_gene_protein") + } + } else { + selected_keys <- intersect(as.character(tables), names(table_specs)) + } + + if (!length(selected_keys)) { + stop("No requested tables were found in the DuckDB.") + } + + phenotype_wide <- .preserve_export_id_text(build_amr_wide()) + exported <- character(0) + + for (key in selected_keys) { + spec <- table_specs[[key]] + + if (key == "amr_phenotype_wide") { + if (is.null(phenotype_wide)) { + if (isTRUE(verbose)) message("Skipping amr_phenotype_wide: no AMR source table found.") + next + } + write_one(phenotype_wide, spec$stem) + exported <- c(exported, spec$stem) + if (isTRUE(verbose)) message("Exported: ", spec$stem) + next + } + + if (is.null(spec$source) || !(spec$source %in% available_tables)) { + if (isTRUE(verbose)) message("Skipping missing table: ", key) + next + } + + df <- .preserve_export_id_text(read_tbl(spec$source)) + out_stem <- spec$stem + + if (identical(amr_phenotype_mode, "append") && + isTRUE(spec$appendable) && + !is.null(phenotype_wide) && + "genome_id" %in% names(df)) { + df <- dplyr::left_join(df, phenotype_wide, by = "genome_id") + df <- .preserve_export_id_text(df) + out_stem <- paste0(spec$stem, "_with_phenotypes") + } + + write_one(df, out_stem) + exported <- c(exported, out_stem) + + if (isTRUE(verbose)) message("Exported: ", out_stem) + } + + invisible(list( + duckdb_path = duckdb_path, + output_path = output_path, + tables = exported, + amr_phenotype_mode = amr_phenotype_mode, + export_formats = export_formats, + export_sequences = isTRUE(export_sequences) + )) +} diff --git a/inst/extdata/Campy_testdata.zip b/inst/extdata/Campy_testdata.zip new file mode 100644 index 0000000..32a4e05 Binary files /dev/null and b/inst/extdata/Campy_testdata.zip differ diff --git a/man/dot-apply_metadata_qc.Rd b/man/dot-apply_metadata_qc.Rd new file mode 100644 index 0000000..d7f7622 --- /dev/null +++ b/man/dot-apply_metadata_qc.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_curation.R +\name{.apply_metadata_qc} +\alias{.apply_metadata_qc} +\title{Apply QC filters to BV-BRC genome metadata} +\usage{ +.apply_metadata_qc( + genome_tbl, + max_checkm_contam = 5, + min_checkm_complete = 95, + gc_deviations = NULL, + length_deviations = NULL, + cds_deviations = NULL +) +} +\arguments{ +\item{genome_tbl}{A tibble of genome metadata containing BV-BRC genome columns.} + +\item{max_checkm_contam}{Numeric scalar. Maximum allowed CheckM contamination (\%).} + +\item{min_checkm_complete}{Numeric scalar. Minimum allowed CheckM completeness (\%).} + +\item{gc_deviations}{Optional numeric scalar. Maximum SDs from the median GC content.} + +\item{length_deviations}{Optional numeric scalar. Maximum SDs from the median genome length.} + +\item{cds_deviations}{Optional numeric scalar. Maximum SDs from the median CDS count.} +} +\value{ +A list with components: +\describe{ +\item{qc_tbl}{QC-annotated genome table} +\item{keep_ids}{Character vector of retained genome IDs} +\item{rejections}{Tibble describing dropped genomes and reasons} +} +} +\description{ +Quality control step with default values for contamination and completeness. +Optional filters for other genome stats to find genomes that are X deviations +away from the median values for those stats. This shouldn't be applied to +jobs that use more than one diverse taxon at a time though! Requires a fairly +normal distribution like you'd see for (most) single species +Returns the cleaned table, retained genome IDs, and a rejection log. +} +\details{ +CheckM contamination and completeness are treated as the primary QC gate. +Deviation-based filters are optional and are best suited to single-taxon runs. +} +\keyword{internal} diff --git a/man/dot-ftpes_download_two_pass.Rd b/man/dot-ftpes_download_two_pass.Rd deleted file mode 100644 index e15d299..0000000 --- a/man/dot-ftpes_download_two_pass.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/data_curation.R -\name{.ftpes_download_two_pass} -\alias{.ftpes_download_two_pass} -\title{Helps manage FTPS downloading from BV-BRC, tryng a quick download first, and -if that fails, trying a longer timeout 2nd pass at the end in case it was a -hiccup. If 2nd pass fails, log and give up on that file.} -\usage{ -.ftpes_download_two_pass( - genome_ids, - out_dir, - workers_first = 4L, - workers_second = 4L, - log_file = NULL -) -} -\description{ -Helps manage FTPS downloading from BV-BRC, tryng a quick download first, and -if that fails, trying a longer timeout 2nd pass at the end in case it was a -hiccup. If 2nd pass fails, log and give up on that file. -} -\keyword{internal} diff --git a/man/dot-ftpes_download_one.Rd b/man/dot-ftps_download_one.Rd similarity index 88% rename from man/dot-ftpes_download_one.Rd rename to man/dot-ftps_download_one.Rd index b89f5d4..af5861d 100644 --- a/man/dot-ftpes_download_one.Rd +++ b/man/dot-ftps_download_one.Rd @@ -1,11 +1,11 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/data_curation.R -\name{.ftpes_download_one} -\alias{.ftpes_download_one} +\name{.ftps_download_one} +\alias{.ftps_download_one} \title{Helps appropriately interface with BV-BRC FTPS server, and avoids getting stuck when malformed files can hang an FTPS connection by introducing safeguards} \usage{ -.ftpes_download_one( +.ftps_download_one( genomeID, out_dir, connect_timeout = 10L, diff --git a/man/dot-ftps_download_two_pass.Rd b/man/dot-ftps_download_two_pass.Rd new file mode 100644 index 0000000..0acd93f --- /dev/null +++ b/man/dot-ftps_download_two_pass.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_curation.R +\name{.ftps_download_two_pass} +\alias{.ftps_download_two_pass} +\title{Helps manage FTPS downloading from BV-BRC, trying a quick download first, and +if that fails, trying a longer timeout 2nd pass at the end in case it was a +hiccup. If the 2nd pass fails, log and give up on that file.} +\usage{ +.ftps_download_two_pass( + genome_ids, + out_dir, + workers_first = 8L, + workers_second = 8L, + log_file = NULL +) +} +\description{ +Helps manage FTPS downloading from BV-BRC, trying a quick download first, and +if that fails, trying a longer timeout 2nd pass at the end in case it was a +hiccup. If the 2nd pass fails, log and give up on that file. +} +\keyword{internal} diff --git a/man/dot-parse_bvbrc_tsv.Rd b/man/dot-parse_bvbrc_tsv.Rd new file mode 100644 index 0000000..049d0d8 --- /dev/null +++ b/man/dot-parse_bvbrc_tsv.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_curation.R +\name{.parse_bvbrc_tsv} +\alias{.parse_bvbrc_tsv} +\title{Parse BV-BRC TSV output} +\usage{ +.parse_bvbrc_tsv(x) +} +\arguments{ +\item{x}{Character vector of output lines from a BV-BRC CLI command.} +} +\value{ +A tibble with minimal name repair. +} +\description{ +Safely cleans blank links and weird formatting for a safe merge of the BV-BRC +TSV data into a tibble. +} +\keyword{internal} diff --git a/man/exportProcessedData.Rd b/man/exportProcessedData.Rd new file mode 100644 index 0000000..c8f16d4 --- /dev/null +++ b/man/exportProcessedData.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_processing.R +\name{exportProcessedData} +\alias{exportProcessedData} +\title{Export processed tables from DuckDB database} +\usage{ +exportProcessedData( + duckdb_path, + output_path = NULL, + amr_phenotype_mode = c("separate", "append"), + export_formats = c("csv"), + export_sequences = FALSE, + tables = NULL, + verbose = TRUE +) +} +\arguments{ +\item{duckdb_path}{Character. Path to the DuckDB database created by the +workflow (for example, \code{Sar.duckdb}).} + +\item{output_path}{Character or NULL. Directory for exports. Defaults to +file.path(dirname(duckdb_path), "processed_exports").} + +\item{amr_phenotype_mode}{Character. One of "separate" or "append". +"separate" exports the AMR labels as a separate wide table. +"append" joins those labels onto the main feature tables before export.} + +\item{export_formats}{Character vector. Any of "csv", "tsv", "parquet", "xlsx".} + +\item{tables}{Character vector or NULL. Tables to export. If NULL, exports the +standard processed tables present in the database.} + +\item{verbose}{Logical. If TRUE, prints progress messages.} +} +\value{ +Invisibly returns a list containing the export path, table names, and mode. +} +\description{ +Reads tables from the DuckDB database produced by the \code{runDataProcessing()} workflow +and exports them as CSV, TSV, Parquet, and/or XLSX. This is an optional step +that allows users to take their processed data outside our amR workflow for +use in their own custom analyses. This is not required to run \code{amRml}! +} diff --git a/man/exportTables.Rd b/man/exportTables.Rd new file mode 100644 index 0000000..4fffb78 --- /dev/null +++ b/man/exportTables.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_curation.R +\name{exportTables} +\alias{exportTables} +\title{Export DuckDB tables and optionally load them into R} +\usage{ +exportTables( + duckdb_path, + output_dir = NULL, + tables = NULL, + skip_tables = c(NULL), + include_summary = TRUE, + export_tables = TRUE, + load_tables = FALSE, + verbose = TRUE +) +} +\arguments{ +\item{duckdb_path}{Character. Path to the DuckDB file.} + +\item{output_dir}{Character or NULL. Directory for exports. Defaults to +file.path(dirname(duckdb_path), "exports").} + +\item{tables}{Character vector or NULL. Tables to export. If NULL, exports +all tables in the database.} + +\item{skip_tables}{Character vector of table names to exclude.} + +\item{include_summary}{Logical. If TRUE, writes summary.csv and summary.txt.} + +\item{load_tables}{Logical. If TRUE, also return the exported tables as +in-memory R data frames.} + +\item{verbose}{Logical. If TRUE, prints progress messages.} +} +\value{ +Invisibly returns a list with exported file paths and, if requested, +in-memory tables. +} +\description{ +Writes selected DuckDB tables to CSV files and optionally returns them as +in-memory R data frames. +} diff --git a/man/genomeList.Rd b/man/genomeList.Rd index dfa6f39..ea36ef2 100644 --- a/man/genomeList.Rd +++ b/man/genomeList.Rd @@ -4,7 +4,7 @@ \alias{genomeList} \title{Build a table of local genome file paths and write to DuckDB} \usage{ -genomeList(base_dir = ".", user_bacs, verbose = TRUE) +genomeList(base_dir = ".", user_bacs, expected_ids = NULL, verbose = TRUE) } \arguments{ \item{base_dir}{Character. Project root.} diff --git a/man/prepareGenomes.Rd b/man/prepareGenomes.Rd index 29edd07..0f7814a 100644 --- a/man/prepareGenomes.Rd +++ b/man/prepareGenomes.Rd @@ -10,13 +10,22 @@ prepareGenomes( base_dir = ".", method = c("ftp", "cli"), overwrite = FALSE, + num_workers = 8L, evidence_mode = c("lab_only", "lab_or_comp", "comp_only", "any"), + max_checkm_contam = 5, + min_checkm_complete = 95, + gc_deviations = NULL, + length_deviations = NULL, + cds_deviations = NULL, + export_tables = FALSE, + load_tables = FALSE, + debug = FALSE, verbose = TRUE ) } \arguments{ \item{user_bacs}{Character vector. Species and/or taxon IDs (e.g. -\code{c("Shigella flexneri", "623")}).} +\code{c("Shigella sonnei", "624")}).} \item{genome_id_file}{Character or NULL. Optional path to a file listing genome IDs (one per line), passed through to \code{retrieveMetadata()}. If provided, the @@ -31,10 +40,25 @@ metadata step is restricted to these genome IDs instead of resolving them from \item{overwrite}{Logical. Passed to metadata filtering and DuckDB creation. Default FALSE.} +\item{num_workers}{Integer. Parallel workers used for genome download. +Applied to both FTP and CLI download branches. Default: 8.} + \item{evidence_mode}{Character. Sets what types of AMR evidence is acceptable. Default \code{lab_only}. \code{any} will not require AMR data for downloads. This will return very large download lists for many species!} +\item{max_checkm_contam}{Numeric scalar. Maximum allowed CheckM contamination (\%).} + +\item{min_checkm_complete}{Numeric scalar. Minimum allowed CheckM completeness (\%).} + +\item{gc_deviations}{Optional numeric scalar. Maximum SDs from the median GC content.} + +\item{length_deviations}{Optional numeric scalar. Maximum SDs from the median genome length.} + +\item{cds_deviations}{Optional numeric scalar. Maximum SDs from the median CDS count.} + +\item{debug}{Logical. If TRUE, keep QC columns in metadata tables.} + \item{verbose}{Logical. Print progress messages. Default TRUE.} } \value{ diff --git a/man/retrieveGenomes.Rd b/man/retrieveGenomes.Rd index 1800b5d..f9d202b 100644 --- a/man/retrieveGenomes.Rd +++ b/man/retrieveGenomes.Rd @@ -10,9 +10,9 @@ retrieveGenomes( method = c("ftp", "cli"), image = "danylmb/bvbrc:5.3", skip_existing = TRUE, - ftp_workers = 4L, - cli_fasta_workers = 4L, - cli_gff_workers = 4L, + ftp_workers = 8L, + cli_fasta_workers = 8L, + cli_gff_workers = 8L, chunk_size = 50L, evidence_mode = c("lab_only", "lab_or_comp", "comp_only", "any"), verbose = TRUE @@ -31,9 +31,9 @@ retrieveGenomes( \item{ftp_workers}{Parallel workers for FTP path (default 8).} -\item{cli_fasta_workers}{Parallel chunk containers for FASTA+GTO (default 4).} +\item{cli_fasta_workers}{Parallel chunk containers for FASTA+GTO (default 8).} -\item{cli_gff_workers}{Parallel chunk containers for GFF export (default 4).} +\item{cli_gff_workers}{Parallel chunk containers for GFF export (default 8).} \item{chunk_size}{Genomes per chunk container (default 50).} diff --git a/man/retrieveMetadata.Rd b/man/retrieveMetadata.Rd index bed8547..90d8cdb 100644 --- a/man/retrieveMetadata.Rd +++ b/man/retrieveMetadata.Rd @@ -12,6 +12,14 @@ retrieveMetadata( abx = "All", overwrite = FALSE, image = "danylmb/bvbrc:5.3", + max_checkm_contam = 5, + min_checkm_complete = 95, + gc_deviations = NULL, + length_deviations = NULL, + cds_deviations = NULL, + debug = FALSE, + export_tables = FALSE, + load_tables = FALSE, verbose = TRUE ) } @@ -34,7 +42,19 @@ Default NULL.} \item{image}{Character. Docker image. Default "danylmb/bvbrc:5.3".} -\item{verbose}{Logical. If TRUE, prints concise messages.} +\item{max_checkm_contam}{Numeric scalar. Maximum allowed CheckM contamination (\%).} + +\item{min_checkm_complete}{Numeric scalar. Minimum allowed CheckM completeness (\%).} + +\item{gc_deviations}{Optional numeric scalar. Maximum SDs from the median GC content.} + +\item{length_deviations}{Optional numeric scalar. Maximum SDs from the median genome length.} + +\item{cds_deviations}{Optional numeric scalar. Maximum SDs from the median CDS count.} + +\item{debug}{Logical. If TRUE, retain \code{metadata_full} and QC columns for inspection.} + +\item{verbose}{Logical. If TRUE, print progress messages.} } \value{ A list with: