Downstream analysis: recommended defaults, clearer labels and decision plots

.gitattributes

@@ -0,0 +1,3 @@
+# MS data files carry byte offsets (indexed mzML); never convert line endings.
+*.mzML -text
+*.fasta -text

app.py

@@ -25,18 +25,21 @@
             st.Page(Path("content", "workflow_configure.py"), title="Configure", icon="⚙️"),
             st.Page(Path("content", "workflow_run.py"), title="Run", icon="🚀"),
         ],
-        "Results": [
+        "Workflow Results": [
             st.Page(Path("content", "results_database_search.py"), title="Database Search", icon="🔬"),
             st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="📈"),
             st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"),
-            st.Page(Path("content", "results_abundance.py"), title="Abundance", icon="📋"),
-
+            st.Page(Path("content", "results_abundance.py"), title="Protein Abundance", icon="📋"),
         ],
-        "Differential Protein Analysis": [
-            st.Page(Path("content", "filtering.py"), title="Filtering", icon="🧹"),
-            st.Page(Path("content", "imputation.py"), title="Imputation", icon="🩹"),
-            st.Page(Path("content", "normalization.py"), title="Normalization", icon="⚖️"),
-            st.Page(Path("content", "statistical.py"), title="Statistical", icon="🔢"),
+        # Steps run in order; each one works on the previous step's output.
+        "Downstream Analysis": [
+            st.Page(Path("content", "filtering.py"), title="1. Filtering", icon="🧹"),
+            st.Page(Path("content", "imputation.py"), title="2. Imputation", icon="🩹"),
+            st.Page(Path("content", "normalization.py"), title="3. Normalization", icon="⚖️"),
+            st.Page(Path("content", "statistical.py"), title="4. Statistics", icon="🔢"),
+        ],
+        # Every plot reads the latest Downstream Analysis output.
+        "Downstream Plots": [
             st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="🌋"),
             st.Page(Path("content", "results_pca.py"), title="PCA", icon="📊"),
             st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="🔥"),

content/enrichment.py

@@ -5,7 +5,7 @@
 import polars as pl
 import streamlit as st
 from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_id_column
+from src.common.results_helpers import get_abundance_data, get_id_column, show_pipeline_banner
 # Import GO Enrichment modules from openms_insight engine
 from openms_insight.analysis.enrichment import calculate_go_enrichment
 
@@ -36,9 +36,7 @@
     st.stop()
 
 final_statistics_report = st.session_state["statistics_df"]
-st.info(
-    "🔄 **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step."
-)
+show_pipeline_banner(uses="statistics")
 
 # --- STEP 2: Preprocessing Mapping Key Configuration ---
 # Identify target identifier columns dynamically

content/filtering.py

@@ -4,8 +4,15 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+from src.common.common import page_setup, save_params, show_fig
+from src.common.postprocessing_plots import filter_threshold_curve
+from src.common.results_helpers import (
+    clear_downstream_steps,
+    get_abundance_data,
+    get_id_column,
+    get_sample_group_map,
+    postprocessing_param,
+)
 
 # Import filtering functions from openms_insight package
 from openms_insight.analysis.filter import (
@@ -22,7 +29,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
     return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore")
 
 params = page_setup()
-st.title("Data Filtering")
+st.title("Step 1 of 4: Filtering")
 
 st.markdown(
     """
@@ -74,15 +81,23 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
     metadata_rows, schema={"sample_id": pl.String, "group": pl.String}
 )
 
-# User selection for filtering strategy
+# User selection for filtering strategy. The default keeps proteins quantified
+# in at least half of the samples of one group, which removes rows that
+# imputation would otherwise have to invent most values for.
+filter_options = ["Low Repeatability", "Low Abundance", "Low Variance"]
 filter_method = st.selectbox(
     "Select Filtering Method",
-    options=["Low Abundance", "Low Repeatability", "Low Variance"],
-    index=0,
-    help="Choose the statistical criteria to prune unreliable protein entries.",
+    options=filter_options,
+    index=filter_options.index(
+        postprocessing_param(params, "postproc-filter-method", filter_options)
+    ),
+    key="postproc-filter-method",
+    help="Choose the statistical criteria to prune unreliable protein entries. "
+    "Recommended: Low Repeatability with at most 50% missing values.",
 )
 
 # Render threshold sliders dynamically based on the selected filter method
+threshold = 0.0  # set by the slider of the selected method below
 if filter_method == "Low Abundance":
     st.markdown(
         "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold."
@@ -91,20 +106,22 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
         "Threshold Percentile (%)",
         min_value=0.0,
         max_value=100.0,
-        value=10.0,
+        value=float(postprocessing_param(params, "postproc-filter-abundance-percentile")),
         step=5.0,
+        key="postproc-filter-abundance-percentile",
     )
 
 elif filter_method == "Low Repeatability":
     st.markdown(
         "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum."
     )
     threshold = st.slider(
-        "Max Missing Ratio",
+        "Max Missing Ratio (%)",
         min_value=0.0,
         max_value=100.0,
-        value=50.0,
+        value=float(postprocessing_param(params, "postproc-filter-max-missing")),
         step=5.0,
+        key="postproc-filter-max-missing",
         help="Allowed missing value (zero or null) ratio per group.",
     )
 
@@ -116,42 +133,64 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
         "Threshold Percentile (%)",
         min_value=0.0,
         max_value=100.0,
-        value=10.0,
+        value=float(postprocessing_param(params, "postproc-filter-variance-percentile")),
         step=5.0,
+        key="postproc-filter-variance-percentile",
     )
 
-# --- SECTION 3: Filter Execution and Collected Results View ---
-if st.button("Apply Filter", type="primary"):
-    # Convert the original Pandas DataFrame into a Polars LazyFrame graph
-    quant_lazy = pl.from_pandas(pivot_df).lazy()
+save_params(params)
+
 
-    # Route execution to the chosen openms_insight engine function
+def run_filter(value: float) -> pl.LazyFrame:
+    """Apply the selected openms_insight filter at threshold ``value``."""
+    quant_lazy = pl.from_pandas(pivot_df).lazy()
     if filter_method == "Low Abundance":
-        filtered_lazy = filter_low_abundance(
+        return filter_low_abundance(
             quantification_data=quant_lazy,
             metadata=metadata_pl,
             group_column="group",
-            threshold_percentile=threshold,
+            threshold_percentile=value,
         )
-    elif filter_method == "Low Repeatability":
+    if filter_method == "Low Repeatability":
         # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5)
-        filtered_lazy = filter_low_repeatability(
-            quantification_data=quant_lazy,
-            metadata=metadata_pl,
-            group_column="group",
-            max_missing_ratio=threshold / 100.0,
-        )
-    else:  # "Low Variance"
-        filtered_lazy = filter_low_variance(
+        return filter_low_repeatability(
             quantification_data=quant_lazy,
             metadata=metadata_pl,
             group_column="group",
-            threshold_percentile=threshold,
+            max_missing_ratio=value / 100.0,
         )
+    return filter_low_variance(
+        quantification_data=quant_lazy,
+        metadata=metadata_pl,
+        group_column="group",
+        threshold_percentile=value,
+    )
+
+
+with st.expander("📈 Help me choose a threshold", expanded=True):
+    st.caption(
+        "How many proteins the selected filter keeps at every threshold. Pick a "
+        "value before the curve drops steeply; flat stretches mean the exact value barely matters."
+    )
+    show_fig(
+        filter_threshold_curve(
+            lambda t: run_filter(t).select(pl.len()).collect().item(),
+            thresholds=[float(t) for t in range(0, 101, 5)],
+            current=threshold,
+            total=pivot_df.shape[0],
+            x_label="Max missing values per group (%)" if filter_method == "Low Repeatability" else "Threshold percentile (%)",
+        ),
+        "filter-threshold-curve",
+    )
+
+# --- SECTION 3: Filter Execution and Collected Results View ---
+if st.button("Apply Filter", type="primary"):
+    filtered_lazy = run_filter(threshold)
 
     # Collect the evaluated lazy graph and convert back to Pandas for visualization
     filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas())
     st.session_state["filtered_df"] = filtered_df
+    clear_downstream_steps("filtered_df")
 
     # Layout response metrics and the filtered matrix
     st.success(f"Successfully applied **{filter_method}** filter!")

content/imputation.py

@@ -4,8 +4,15 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+from src.common.common import page_setup, save_params, show_fig
+from src.common.postprocessing_plots import imputed_value_preview, missingness_vs_intensity
+from src.common.results_helpers import (
+    clear_downstream_steps,
+    get_abundance_data,
+    get_id_column,
+    get_sample_group_map,
+    postprocessing_param,
+)
 
 # Import imputation algorithms from openms_insight engine
 from openms_insight.analysis.imputation import impute_mar, impute_smallest_value
@@ -18,7 +25,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
     return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore")
 
 params = page_setup()
-st.title("Missing Value Imputation")
+st.title("Step 2 of 4: Imputation")
 
 st.markdown(
     """
@@ -83,60 +90,101 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 )
 
 # User selection for core missingness assumption strategy
+# In DDA label-free data most missing values are proteins below the detection
+# limit (MNAR), so the default fills them with the protein's own smallest
+# observed intensity. The global minimum sits far below most proteins and turns
+# a single random dropout into a large false fold change; MAR group imputation
+# cannot fill a protein that is missing in a whole group.
+impute_options = ["MNAR (Missing Not At Random)", "MAR (Missing At Random)"]
 impute_category = st.selectbox(
     "Select Imputation Class",
-    options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"],
-    index=0,
-    help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.",
+    options=impute_options,
+    index=impute_options.index(
+        postprocessing_param(params, "postproc-impute-class", impute_options)
+    ),
+    key="postproc-impute-class",
+    help="MAR uses group metrics (Mean/Median). MNAR fills values below the limit of detection. "
+    "Recommended: MNAR with row scope.",
 )
 
 # Render algorithmic options sub-menus based on the parent selection
+strategy_opt = scope_opt = None  # only the selected class's option is shown
 if impute_category == "MAR (Missing At Random)":
     st.markdown(
         "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group."
     )
+    mar_options = ["median", "mean"]
     strategy_opt = st.radio(
         "Mathematical Strategy",
-        options=["median", "mean"],
-        index=0,
+        options=mar_options,
+        index=mar_options.index(
+            postprocessing_param(params, "postproc-impute-mar-strategy", mar_options)
+        ),
+        key="postproc-impute-mar-strategy",
         horizontal=True,
     )
 
 elif impute_category == "MNAR (Missing Not At Random)":
     st.markdown(
         "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits."
     )
+    scope_options = ["row", "global"]
     scope_opt = st.radio(
         "Detection Minimum Scope",
-        options=["row", "global"],
-        index=0,
+        options=scope_options,
+        index=scope_options.index(
+            postprocessing_param(params, "postproc-impute-mnar-scope", scope_options)
+        ),
+        key="postproc-impute-mnar-scope",
         horizontal=True,
         help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.",
     )
 
-# --- SECTION 3: Imputation Execution ---
-if st.button("Apply Imputation", type="primary"):
-    # Initialize optimization pipeline graph via lazy loading conversion
-    quant_lazy = pl.from_pandas(base_df).lazy()
+save_params(params)
 
-    # Route configuration matrix parameters to designated engine function channels
+
+def run_imputation() -> pl.LazyFrame:
+    """Apply the selected openms_insight imputation to the input table."""
+    quant_lazy = pl.from_pandas(base_df).lazy()
     if impute_category == "MAR (Missing At Random)":
-        imputed_lazy = impute_mar(
+        return impute_mar(
             quantification_data=quant_lazy,
             metadata=metadata_pl,
             group_column="group",
             strategy=strategy_opt,
         )
-    else:  # "MNAR (Missing Not At Random)"
-        imputed_lazy = impute_smallest_value(
-            quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt
+    return impute_smallest_value(
+        quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt
+    )
+
+
+grouped_samples = metadata_pl["sample_id"].to_list()
+with st.expander("📈 Help me choose a method", expanded=True):
+    if not grouped_samples:
+        st.info("Assign sample groups in Configure to see these plots.")
+    else:
+        st.caption(
+            "Mean intensity of proteins with and without missing values. If the "
+            "orange curve sits clearly lower, values fall below the detection limit (choose MNAR); if the curves overlap, dropout is random (MAR fits)."
+        )
+        show_fig(missingness_vs_intensity(base_df, grouped_samples), "imputation-missingness")
+        st.caption(
+            "Where the current setting places the filled-in values (red) relative to "
+            "observed ones (blue). They should sit at the low edge of the observed values; a separate spike far below them inflates fold changes."
         )
+        preview_df = run_imputation().collect().to_pandas()
+        show_fig(imputed_value_preview(base_df, preview_df, grouped_samples), "imputation-preview")
+
+# --- SECTION 3: Imputation Execution ---
+if st.button("Apply Imputation", type="primary"):
+    imputed_lazy = run_imputation()
 
     # Resolve lazy graph optimization tree and push to display data frame structure
     imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas())
 
     # 💾 Save current output into Session State for down-stream processing (Normalization, Statistics)
     st.session_state["imputed_df"] = imputed_df
+    clear_downstream_steps("imputed_df")
 
     st.success(f"Successfully finalized **{impute_category}** imputation step!")