diff --git a/app.py b/app.py index cdaf2d6..3e3b047 100644 --- a/app.py +++ b/app.py @@ -30,6 +30,7 @@ st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="📈"), st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"), st.Page(Path("content", "results_abundance.py"), title="Protein Abundance", icon="📋"), + st.Page(Path("content", "results_download.py"), title="Download", icon="⬇️"), ], # Steps run in order; each one works on the previous step's output. "Downstream Analysis": [ diff --git a/content/results_abundance.py b/content/results_abundance.py index 38c42bc..4c98cb5 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,11 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd -import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data from src.workflow.ParameterManager import ParameterManager +from src.common.insight_tables import show_insight_table params = page_setup() st.title("Abundance Quantification") @@ -40,6 +40,8 @@ st.stop() csv_file = csv_files[0] +cache_dir = workflow_dir / "results" / "insight_cache" +table_sources = [csv_file, workflow_dir / "params.json"] def render_protein_table(pivot_df, is_lfq=True): """Common function to render the protein-level abundance table""" @@ -56,42 +58,27 @@ def render_protein_table(pivot_df, is_lfq=True): exclude_cols = [id_col, "PeptideSequence"] sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] - help_text = "Raw sample intensities" - y_min = None + display_cols = [id_col] + sample_cols + ["PeptideSequence"] else: # Handle non-LFQ mode columns (Log2-transformed Intensity) id_col = "protein" exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] - pivot_df["Intensity"] = pivot_df[sample_cols].apply( - lambda row: [np.log2(v + 1) for v in row], axis=1 - ) - display_cols = [id_col, "Intensity"] + sample_cols - help_text = "Sample intensities (log2 scale)" - y_min = 0 + display_cols = [id_col] + sample_cols # Filter to available columns, then sort and display available_cols = [c for c in display_cols if c in pivot_df.columns] view_df = pivot_df[available_cols] - st.dataframe( + show_insight_table( view_df, - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help=help_text, - width="small", - y_min=y_min, - ), - }, - use_container_width=True, + name="abundance_protein", + cache_dir=cache_dir, + source_files=table_sources, + title="protein_abundance", ) -protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - try: df = pd.read_csv(csv_file) @@ -119,7 +106,13 @@ def render_protein_table(pivot_df, is_lfq=True): "peptide sequences, charge states, and intensities across samples. " "Each row represents one peptide-spectrum match detected from the MS/MS analysis." ) - st.dataframe(df, use_container_width=True) + show_insight_table( + df, + name="abundance_psm", + cache_dir=cache_dir, + source_files=table_sources, + title="psm_quantification", + ) else: pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) @@ -132,7 +125,13 @@ def render_protein_table(pivot_df, is_lfq=True): with pre_processing_tab: st.write("### Final Results (Intensity matrix)") - st.dataframe(pivot_df.head(10)) + show_insight_table( + pivot_df, + name="abundance_intensity_matrix", + cache_dir=cache_dir, + source_files=table_sources, + title="intensity_matrix", + ) with protein_tab: render_protein_table(pivot_df, is_lfq=False) diff --git a/content/results_download.py b/content/results_download.py new file mode 100644 index 0000000..476f382 --- /dev/null +++ b/content/results_download.py @@ -0,0 +1,21 @@ +"""Download Results Page.""" +import streamlit as st +from src.common.common import page_setup +from src.WorkflowTest import WorkflowTest + +params = page_setup() + +wf = WorkflowTest() + +st.title("Download Results") + +st.markdown( + """ +Every file the workflow produced: identifications (idXML) from each search step, +quantification tables, the spectral library if one was built, and the exports in +**exports/**: all PSMs per run (`*_psms.tsv`) and every PSM's matched fragment +ions per scan (`*_fragment_ions.tsv`). +""" +) + +wf.show_results_download_section(exclude=["insight_cache"]) diff --git a/content/results_filtered.py b/content/results_filtered.py index bb97018..d6f31aa 100644 --- a/content/results_filtered.py +++ b/content/results_filtered.py @@ -26,7 +26,7 @@ st.stop() workflow_dir = get_workflow_dir(st.session_state["workspace"]) -filter_dir = workflow_dir / "results" / "filter_results" +filter_dir = workflow_dir / "results" / "psm_filter" cache_dir = workflow_dir / "results" / "insight_cache" if not filter_dir.exists(): diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index c98cc06..30d19b4 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -15,7 +15,7 @@ from src.workflow.WorkflowManager import WorkflowManager from src.common.common import page_setup from src.common.results_helpers import get_abundance_data -from src.common.results_helpers import parse_idxml, build_spectra_cache, load_idxml +from src.common.results_helpers import parse_idxml, build_spectra_cache, load_idxml, write_psm_exports from openms_insight import Table, Heatmap, LinePlot, SequenceView # Accession prefixes that mark decoy entries in a FASTA database. @@ -876,25 +876,21 @@ def execution(self) -> bool: # Get mzML directory mzml_dir = Path(in_mzML[0]).parent - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} + # Build spectra cache (once, shared by all stages). Every input mzML + # gets a fixed file index, so each PSM links to its own run's spectra. + filename_to_index = {Path(m).name: i for i, m in enumerate(in_mzML)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index, in_mzML) for idxml_file in comet_results: idxml_path = Path(idxml_file) cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - # Initialize Table component (caches itself) Table( cache_id=f"table_{cache_id_prefix}", @@ -990,7 +986,7 @@ def execution(self) -> bool: cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue @@ -1083,7 +1079,7 @@ def execution(self) -> bool: cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue @@ -1154,6 +1150,10 @@ def execution(self) -> bool: }, ) + # Downloadable exports: every PSM, and every PSM's matched + # fragment ions against its own spectrum + write_psm_exports(id_df, seq_view, results_dir_path / "exports", cache_id_prefix) + self.logger.log("✅ Filtering complete") # if not Path(filter_results[i]).exists(): @@ -1389,25 +1389,21 @@ def execution(self) -> bool: # Get mzML directory mzml_dir = Path(in_mzML[0]).parent - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} + # Build spectra cache (once, shared by all stages). Every input mzML + # gets a fixed file index, so each PSM links to its own run's spectra. + filename_to_index = {Path(m).name: i for i, m in enumerate(in_mzML)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index, in_mzML) for idxml_file in comet_results: idxml_path = Path(idxml_file) cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - # Initialize Table component (caches itself) Table( cache_id=f"table_{cache_id_prefix}", @@ -1499,7 +1495,7 @@ def execution(self) -> bool: cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue @@ -1589,7 +1585,7 @@ def execution(self) -> bool: cache_id_prefix = idxml_path.stem # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) + id_df, spectra_data = parse_idxml(idxml_path, filename_to_index) if id_df.is_empty(): self.logger.log(f"⚠️ No identifications in {idxml_path.name}, skipping its result views") continue @@ -1660,6 +1656,10 @@ def execution(self) -> bool: }, ) + # Downloadable exports: every PSM, and every PSM's matched + # fragment ions against its own spectrum + write_psm_exports(id_df, seq_view, results_dir_path / "exports", cache_id_prefix) + # --- IDMapper --- self.logger.log("🗺️ Mapping IDs to isobaric consensus features...") for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): @@ -1910,7 +1910,7 @@ def idxml_to_df(idxml_file): # ================================ with filter_tab: - filter_dir = Path(self.workflow_dir, "results", "filter_results") + filter_dir = Path(self.workflow_dir, "results", "psm_filter") filter_files = sorted(filter_dir.glob("*.idXML")) if not filter_files: diff --git a/src/common/insight_tables.py b/src/common/insight_tables.py new file mode 100644 index 0000000..d197d44 --- /dev/null +++ b/src/common/insight_tables.py @@ -0,0 +1,104 @@ +"""Streamed result tables built on openms-insight. + +``st.dataframe`` ships the whole table to the browser, which stalls on +full-scale runs. An openms-insight ``Table`` caches the data on disk once and +sends one page at a time; its download button exports every row and column. +""" +import hashlib +import shutil +from pathlib import Path + +import pandas as pd +import polars as pl +from openms_insight import Table + +ROW_FIELD = "row" + + +def _cache_key(source_files, extra: str) -> str: + """Hash the source files' identity and mtimes, so new results get a new cache.""" + parts = [extra] + for path in source_files: + path = Path(path) + mtime = path.stat().st_mtime_ns if path.exists() else 0 + parts.append(f"{path}:{mtime}") + return hashlib.sha1("|".join(parts).encode()).hexdigest()[:10] + + +def _to_polars(df: pd.DataFrame) -> tuple[pl.DataFrame, list[dict]]: + """Convert to polars with Tabulator-safe field names and column definitions. + + Tabulator reads a "." in a field name as a nested lookup, so such columns get + a safe field name and keep their original name as the column title. + """ + df = df.reset_index(drop=True) + renames, column_definitions, used = {}, [], {ROW_FIELD} + for column in df.columns: + field = str(column).replace(".", "_") + while field in used: + field = f"{field}_" + used.add(field) + renames[column] = field + col_def = {"field": field, "title": str(column), "headerTooltip": True} + if pd.api.types.is_numeric_dtype(df[column]): + col_def["sorter"] = "number" + col_def["hozAlign"] = "right" + column_definitions.append(col_def) + df = df.rename(columns=renames) + for column in df.columns: + if df[column].dtype == object: + df[column] = df[column].map(lambda v: None if v is None or v != v else str(v)) + pl_df = pl.from_pandas(df).with_row_index(ROW_FIELD) + return pl_df, column_definitions + + +def show_insight_table( + df: pd.DataFrame, + name: str, + cache_dir, + source_files=(), + extra_key: str = "", + title: str | None = None, + height: int = 500, + page_size: int = 100, + **table_kwargs, +) -> None: + """Render ``df`` as a server-side paginated openms-insight table. + + The cache is keyed on ``source_files`` (path and mtime) and ``extra_key``; + when either changes (a rerun, new group assignments) a new cache replaces the + old one, so the page never shows a previous run's rows. + + Args: + df: Table to show. + name: Stable name for this table; must be unique per page. + cache_dir: Directory holding openms-insight caches. + source_files: Files the table is derived from. + extra_key: Anything else the table depends on (e.g. parameters). + title: Title, also used as the CSV download's file name. + height: Component height in pixels. + page_size: Rows per page. + **table_kwargs: Passed on to ``Table`` (e.g. ``initial_sort``). + """ + cache_dir = Path(cache_dir) + cache_id = f"{name}_{_cache_key(source_files, extra_key)}" + + if not (cache_dir / cache_id).is_dir(): + # Drop this table's caches from earlier results before building the new one + for stale in cache_dir.glob(f"{name}_" + "?" * 10): + shutil.rmtree(stale, ignore_errors=True) + cache_dir.mkdir(parents=True, exist_ok=True) + pl_df, column_definitions = _to_polars(df) + Table( + cache_id=cache_id, + data=pl_df.lazy(), + cache_path=str(cache_dir), + column_definitions=column_definitions, + index_field=ROW_FIELD, + title=title or name, + page_size=page_size, + **table_kwargs, + ) + + table = Table(cache_id=cache_id, cache_path=str(cache_dir)) + table(key=f"insight_table_{name}", height=height) diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index 61058ac..c434b23 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -105,12 +105,29 @@ def extract_scan_number(native_id: str) -> int: return int(match.group(1)) if match else 0 -def extract_filename_from_idxml(idxml_path: Path) -> str: - """Derive mzML filename from idXML filename.""" +def extract_filename_from_idxml(idxml_path: Path, mzml_names=None) -> str: + """Derive the source mzML filename from an idXML filename. + + The workflow names idXML files ``_...``, e.g. + ``02COVID_comet_perc_filter.idXML``. When the input mzML names are known the + longest one whose stem prefixes the idXML stem wins; otherwise the known + step suffixes are stripped from the end. + """ stem = idxml_path.stem - for suffix in ['_comet', '_per', '_filter']: - stem = stem.replace(suffix, '') - return f"{stem}.mzML" + if mzml_names: + candidates = [ + name for name in mzml_names + if stem == Path(name).stem or stem.startswith(f"{Path(name).stem}_") + ] + if candidates: + return max(candidates, key=len) + while True: + for suffix in ['_filter', '_perc', '_per', '_comet']: + if stem.endswith(suffix): + stem = stem[:-len(suffix)] + break + else: + return f"{stem}.mzML" ID_DF_SCHEMA = { @@ -127,19 +144,25 @@ def extract_filename_from_idxml(idxml_path: Path) -> str: } -def parse_idxml(idxml_path: Path) -> tuple[pl.DataFrame, list[str]]: +def parse_idxml(idxml_path: Path, filename_to_index: dict | None = None) -> tuple[pl.DataFrame, list[str]]: """Parse idXML and return DataFrame for openms_insight. + Args: + idxml_path: idXML file of one run. + filename_to_index: Maps every input mzML filename to its file index in + the spectra cache. The PSMs get the index of their own run, so they + link to that run's spectra. + Returns: Tuple of (id_df, spectra_data list of source filenames) """ proteins, peptides = load_idxml(idxml_path) # Derive mzML filename from idXML filename (e.g., 02COVID_filter.idXML -> 02COVID.mzML) - spectra_data = [extract_filename_from_idxml(idxml_path)] + source = extract_filename_from_idxml(idxml_path, list(filename_to_index or [])) + spectra_data = [source] - # Build filename to index mapping - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + run_index = (filename_to_index or {}).get(source, 0) records = [] for pep in peptides: @@ -151,9 +174,9 @@ def parse_idxml(idxml_path: Path) -> tuple[pl.DataFrame, list[str]]: spec_ref = spec_ref.decode() scan_id = extract_scan_from_ref(spec_ref) - # Get file index from id_merge_index or derive from filename - file_index = pep.getMetaValue("id_merge_index") if pep.metaValueExists("id_merge_index") else 0 - filename = spectra_data[file_index] if file_index < len(spectra_data) else "" + # One idXML holds one run, so every PSM belongs to that run's file index + file_index = run_index + filename = spectra_data[0] for h in pep.getHits(): records.append({ @@ -174,12 +197,14 @@ def parse_idxml(idxml_path: Path) -> tuple[pl.DataFrame, list[str]]: return pl.DataFrame(records, schema=ID_DF_SCHEMA), spectra_data -def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.DataFrame, dict]: +def build_spectra_cache(mzml_dir: Path, filename_to_index: dict, mzml_files=None) -> tuple[pl.DataFrame, dict]: """Extract MS2 spectra from mzML files and return DataFrame. Args: mzml_dir: Directory containing mzML files filename_to_index: Dict mapping filename to file_index + mzml_files: mzML paths to read. Defaults to every mzML in ``mzml_dir``; + pass the workflow inputs so referenced (not copied) files are found. Returns: Tuple of (spectra_df, updated filename_to_index) @@ -187,7 +212,8 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat records = [] peak_id = 0 - for mzml_path in sorted(mzml_dir.glob("*.mzML")): + paths = [Path(f) for f in mzml_files] if mzml_files else sorted(mzml_dir.glob("*.mzML")) + for mzml_path in paths: # Get or create file index if mzml_path.name not in filename_to_index: filename_to_index[mzml_path.name] = len(filename_to_index) @@ -215,6 +241,31 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat return pl.DataFrame(records), filename_to_index +def write_psm_exports(id_df: pl.DataFrame, seq_view, export_dir: Path, prefix: str) -> None: + """Write one run's PSM table and per-scan fragment ion table as TSV. + + ``_psms.tsv`` holds every PSM with all its columns. + ``_fragment_ions.tsv`` holds, for every PSM, each fragment ion the + SequenceView matched in its spectrum (same ion types, losses and tolerance + as the view), together with the PSM's run, RT, m/z, score and proteins. + """ + export_dir.mkdir(parents=True, exist_ok=True) + id_df.write_csv(export_dir / f"{prefix}_psms.tsv", separator="\t") + + fragments = seq_view.export_fragment_ions() + if fragments.height and "sequence_id" in fragments.columns and "id_idx" in id_df.columns: + psm_info = id_df.select( + pl.col("id_idx").cast(fragments.schema["sequence_id"]).alias("sequence_id"), + "filename", + "rt", + pl.col("mz").alias("precursor_mz"), + "score", + "protein_accession", + ) + fragments = fragments.join(psm_info, on="sequence_id", how="left") + fragments.write_csv(export_dir / f"{prefix}_fragment_ions.tsv", separator="\t") + + @st.cache_data def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: """Load CSV and build abundance matrices for downstream preprocessing. diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index b47fdc9..91bb282 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -1670,6 +1670,123 @@ def _show_queue_status(self, status: dict) -> None: def results_section(self, custom_results_function) -> None: custom_results_function() + def results_download_section( + self, exclude: list[str] | None = None, results_dir: str | Path | None = None + ) -> None: + """ + Offer every file under the workflow's results directory for download. + + Lists the files, lets the user download any single one, and builds a + ZIP of the chosen result folders on request. Only the selected file (or + the ZIP) is read into memory, so large result sets stay cheap to render. + + Args: + exclude: Names of files or folders to leave out (matched against + every part of a file's path relative to the results directory, + glob patterns allowed), e.g. caches: ``["insight_cache", "*.tmp"]``. + results_dir: Directory to offer. Defaults to ``/results``. + """ + from fnmatch import fnmatch + + results_dir = Path(results_dir or Path(self.workflow_dir, "results")) + exclude = list(exclude or []) + + def excluded(rel: Path) -> bool: + return any(fnmatch(part, pattern) for part in rel.parts for pattern in exclude) + + files = [] + if results_dir.exists(): + for path in sorted(results_dir.rglob("*")): + rel = path.relative_to(results_dir) + if path.is_file() and not excluded(rel): + files.append(rel) + + if not files: + st.info("No results to download yet. Run the workflow first.") + return + + def folder_of(rel: Path) -> str: + return rel.parts[0] if len(rel.parts) > 1 else "." + + def human_size(n_bytes: float) -> str: + for unit in ["B", "KB", "MB", "GB"]: + if n_bytes < 1024 or unit == "GB": + return f"{n_bytes:.0f} {unit}" if unit == "B" else f"{n_bytes:.1f} {unit}" + n_bytes /= 1024 + + st.dataframe( + [ + { + "Folder": folder_of(rel), + "File": str(rel.relative_to(folder_of(rel))), + "Size": human_size(Path(results_dir, rel).stat().st_size), + "Modified": datetime.fromtimestamp( + Path(results_dir, rel).stat().st_mtime + ).strftime("%Y-%m-%d %H:%M"), + } + for rel in files + ], + hide_index=True, + width="stretch", + ) + + key_prefix = f"results-download-{Path(self.workflow_dir).name}" + c1, c2 = st.columns(2) + + with c1: + st.markdown("**Single file**") + selected = st.selectbox( + "File", + files, + format_func=str, + key=f"{key_prefix}-file", + label_visibility="collapsed", + ) + if selected is not None: + st.download_button( + "⬇️ Download file", + data=Path(results_dir, selected).read_bytes(), + file_name=Path(selected).name, + key=f"{key_prefix}-file-button", + width="stretch", + ) + + with c2: + st.markdown("**ZIP archive**") + folders = sorted({folder_of(rel) for rel in files}) + chosen = st.multiselect( + "Folders", + folders, + default=folders, + key=f"{key_prefix}-folders", + label_visibility="collapsed", + ) + zip_path = Path(self.workflow_dir, "downloads", "results.zip") + prepared_key = f"{key_prefix}-prepared" + if st.button( + "📦 Prepare ZIP", + disabled=not chosen, + key=f"{key_prefix}-zip", + width="stretch", + ): + zip_path.parent.mkdir(parents=True, exist_ok=True) + selected_files = [rel for rel in files if folder_of(rel) in chosen] + with st.spinner(f"Compressing {len(selected_files)} files..."): + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_file: + for rel in selected_files: + zip_file.write(Path(results_dir, rel), Path("results", rel)) + st.session_state[prepared_key] = sorted(chosen) + if st.session_state.get(prepared_key) == sorted(chosen) and zip_path.exists(): + with open(zip_path, "rb") as f: + st.download_button( + "⬇️ Download ZIP", + data=f, + file_name=f"{Path(self.workflow_dir).name}-results.zip", + mime="application/zip", + key=f"{key_prefix}-zip-button", + width="stretch", + ) + def non_default_params_summary(self): # Display a summary of non-default TOPP parameters and all others (custom and python scripts) diff --git a/src/workflow/WorkflowManager.py b/src/workflow/WorkflowManager.py index 856ae56..fd24c47 100644 --- a/src/workflow/WorkflowManager.py +++ b/src/workflow/WorkflowManager.py @@ -258,6 +258,14 @@ def show_results_section(self) -> None: """ self.ui.results_section(self.results) + def show_results_download_section(self, exclude: list[str] | None = None) -> None: + """ + Shows a page offering every file in the workflow's results directory for + download, singly or as a ZIP. Pass ``exclude`` to hide folders or files + that are not results, such as visualization caches. + """ + self.ui.results_download_section(exclude=exclude) + def upload(self) -> None: """ Add your file upload widgets here diff --git a/tests/test_result_exports.py b/tests/test_result_exports.py new file mode 100644 index 0000000..5cde0c9 --- /dev/null +++ b/tests/test_result_exports.py @@ -0,0 +1,98 @@ +"""Tests for the PSM / fragment ion exports and the streamed result tables.""" +from pathlib import Path + +import pandas as pd +import polars as pl +import pytest + +openms_insight = pytest.importorskip("openms_insight") +from openms_insight import SequenceView # noqa: E402 + +if not hasattr(SequenceView, "export_fragment_ions"): + pytest.skip("needs openms-insight with SequenceView.export_fragment_ions", allow_module_level=True) + +from src.common.results_helpers import write_psm_exports # noqa: E402 + +PROTON = 1.007276 + + +def _y_ion_mz(sequence: str, n: int, charge: int = 1) -> float: + import pyopenms as poms + + neutral = poms.AASequence.fromString(sequence).getSuffix(n).getMonoWeight() + return (neutral + charge * PROTON) / charge + + +def test_write_psm_exports(tmp_path): + id_df = pl.DataFrame( + { + "id_idx": [0, 1], + "scan_id": [5, 6], + "file_index": [0, 1], + "filename": ["A.mzML", "B.mzML"], + "sequence": ["PEPTIDEK", "ELVISK"], + "charge": [2, 2], + "mz": [465.2, 351.7], + "rt": [100.0, 200.0], + "score": [0.001, 0.002], + "protein_accession": ["P1", "P2"], + } + ) + peaks = pl.DataFrame( + { + "peak_id": [0, 1, 2], + "file_index": [0, 1, 0], + "scan_id": [5, 6, 6], + "mass": [_y_ion_mz("PEPTIDEK", 3), _y_ion_mz("ELVISK", 2), 999.0], + "intensity": [10.0, 20.0, 30.0], + } + ) + seq_view = SequenceView( + cache_id="seqview_test", + sequence_data=id_df.lazy() + .select(["id_idx", "sequence", "charge", "file_index", "scan_id"]) + .rename({"id_idx": "sequence_id", "charge": "precursor_charge"}), + peaks_data=peaks.lazy(), + filters={"identification": "sequence_id", "file": "file_index", "spectrum": "scan_id"}, + cache_path=str(tmp_path / "cache"), + deconvolved=False, + annotation_config={"ion_types": ["b", "y"], "tolerance": 0.02, "tolerance_ppm": False}, + ) + + write_psm_exports(id_df, seq_view, tmp_path / "exports", "run_filter") + + psms = pl.read_csv(tmp_path / "exports" / "run_filter_psms.tsv", separator="\t") + assert psms.height == 2 + assert set(id_df.columns) <= set(psms.columns) + + frags = pl.read_csv(tmp_path / "exports" / "run_filter_fragment_ions.tsv", separator="\t") + matched = {(r["sequence"], r["ion"], r["filename"]) for r in frags.iter_rows(named=True) if r["charge"] == 1} + assert ("PEPTIDEK", "y3", "A.mzML") in matched + assert ("ELVISK", "y2", "B.mzML") in matched + # PSM context comes along with every ion + assert {"rt", "precursor_mz", "score", "protein_accession"} <= set(frags.columns) + + +def test_insight_table_cache_follows_source_file(tmp_path, monkeypatch): + from src.common import insight_tables + + rendered = [] + monkeypatch.setattr(insight_tables.Table, "__call__", lambda self, **kw: rendered.append(self._cache_id)) + + source = tmp_path / "quant.csv" + source.write_text("x") + df = pd.DataFrame({"Protein.Name": ["A", "B"], "sample1": [1.0, 2.0], "note": [None, "x"]}) + + insight_tables.show_insight_table(df, "t", tmp_path / "cache", source_files=[source]) + first = rendered[-1] + table = insight_tables.Table(cache_id=first, cache_path=str(tmp_path / "cache")) + fields = [c["field"] for c in table._column_definitions] + assert fields == ["Protein_Name", "sample1", "note"] + assert table._column_definitions[0]["title"] == "Protein.Name" + + # A rerun rewrites the source: a new cache replaces the old one + import os + os.utime(source, ns=(1, 1)) + insight_tables.show_insight_table(df, "t", tmp_path / "cache", source_files=[source]) + assert rendered[-1] != first + assert not (tmp_path / "cache" / first).exists() diff --git a/tests/test_results_download_section.py b/tests/test_results_download_section.py new file mode 100644 index 0000000..faa2a9f --- /dev/null +++ b/tests/test_results_download_section.py @@ -0,0 +1,70 @@ +"""Tests for StreamlitUI.results_download_section (the workflow Download page).""" + +import zipfile +from pathlib import Path + +from streamlit.testing.v1 import AppTest + + +def _script(workflow_dir: str, exclude: list): + from pathlib import Path + + from src.workflow.StreamlitUI import StreamlitUI + + ui = StreamlitUI.__new__(StreamlitUI) + ui.workflow_dir = Path(workflow_dir) + ui.results_download_section(exclude=exclude) + + +def _run(workflow_dir: Path, exclude=None) -> AppTest: + at = AppTest.from_function(_script, args=(str(workflow_dir), exclude or [])) + return at.run(timeout=30) + + +def _populate(workflow_dir: Path) -> None: + for rel in [ + "quant/proteins.csv", + "quant/nested/peptides.csv", + "ids/run1.idXML", + "insight_cache/table/data.parquet", + "summary.txt", + ]: + path = Path(workflow_dir, "results", rel) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rel) + + +def test_no_results_shows_info(tmp_path): + at = _run(tmp_path) + assert not at.exception + assert "No results to download yet" in at.info[0].value + + +def test_lists_files_and_honours_exclude(tmp_path): + _populate(tmp_path) + at = _run(tmp_path, exclude=["insight_cache"]) + assert not at.exception + + listed = at.dataframe[0].value + assert sorted(zip(listed["Folder"], listed["File"])) == [ + (".", "summary.txt"), + ("ids", "run1.idXML"), + ("quant", "nested/peptides.csv"), + ("quant", "proteins.csv"), + ] + assert "insight_cache" not in set(listed["Folder"]) + assert sorted(at.multiselect[0].value) == [".", "ids", "quant"] + + +def test_zip_contains_chosen_folders(tmp_path): + _populate(tmp_path) + at = _run(tmp_path, exclude=["insight_cache"]) + at.multiselect[0].set_value(["quant"]).run() + at.button[0].click().run() + assert not at.exception + + with zipfile.ZipFile(tmp_path / "downloads" / "results.zip") as zf: + assert sorted(zf.namelist()) == [ + "results/quant/nested/peptides.csv", + "results/quant/proteins.csv", + ] diff --git a/tests/test_results_helpers.py b/tests/test_results_helpers.py index 3d7b669..626dfec 100644 --- a/tests/test_results_helpers.py +++ b/tests/test_results_helpers.py @@ -8,6 +8,7 @@ extract_scan_number, get_workflow_dir, load_idxml, + parse_idxml, ) @@ -28,6 +29,17 @@ def test_extract_filename_from_idxml_strips_suffixes(): assert extract_filename_from_idxml(Path("02COVID_filter.idXML")) == "02COVID.mzML" assert extract_filename_from_idxml(Path("sample_comet.idXML")) == "sample.mzML" assert extract_filename_from_idxml(Path("run_per.idXML")) == "run.mzML" + # TMT names its Percolator/IDFilter output _comet_perc[_filter] + assert extract_filename_from_idxml(Path("S1_comet_perc_filter.idXML")) == "S1.mzML" + assert extract_filename_from_idxml(Path("S1_comet_perc.idXML")) == "S1.mzML" + + +def test_extract_filename_from_idxml_prefers_input_names(): + names = ["run_a.mzML", "run_a_percent.mzML", "sample.mzML"] + # A stem that itself ends like a step suffix still resolves to its input + assert extract_filename_from_idxml(Path("run_a_percent_comet.idXML"), names) == "run_a_percent.mzML" + assert extract_filename_from_idxml(Path("run_a_per.idXML"), names) == "run_a.mzML" + assert extract_filename_from_idxml(Path("sample_comet_perc_filter.idXML"), names) == "sample.mzML" def _write_idxml(path): @@ -98,3 +110,17 @@ def test_load_idxml_accepts_str_path(tmp_path): _write_idxml(idxml) assert len(load_idxml(str(idxml))[1]) == 2 + + +def test_parse_idxml_assigns_each_run_its_own_file_index(tmp_path): + """PSMs link to their own run's spectra, not always to the first run's.""" + filename_to_index = {"A.mzML": 0, "B.mzML": 1} + idxml = tmp_path / "B_comet_perc_filter.idXML" + _write_idxml(idxml) + + id_df, spectra_data = parse_idxml(idxml, filename_to_index) + + assert spectra_data == ["B.mzML"] + assert id_df["file_index"].to_list() == [1, 1] + assert id_df["filename"].to_list() == ["B.mzML", "B.mzML"] + assert id_df["scan_id"].to_list() == [1, 2]