Add support for atom reachability integration

atom.Dockerfile

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+FROM ghcr.io/appthreat/atom:v3.2.0
+
+RUN set -e; \
+    JAVA_DIR="$(ls -d /opt/java/*-tem | head -n1)"; \
+    MAVEN_DIR="$(ls -d /opt/maven/* | head -n1)"; \
+    SBT_DIR="$(ls -d /opt/sbt/* | head -n1)"; \
+    SCALA_DIR="$(ls -d /opt/scala/* | head -n1)"; \
+    mkdir -p /atom-staged/opt /atom-staged/bin /atom-staged/lib/node_modules; \
+    cp -a /opt/wrapper /opt/java /opt/maven /opt/sbt /opt/scala /opt/vendor /atom-staged/opt/; \
+    cp /usr/bin/node-22 /atom-staged/bin/node; \
+    for lib in $(ldd /usr/bin/node-22 | awk '/=>/{print $3}'); do \
+      case "$lib" in */libc.so*|*/libm.so*|*/libpthread*|*/libdl.so*|*/librt*|*/ld-linux*) ;; *) cp -aL "$lib" /atom-staged/lib/ ;; esac; \
+    done; \
+    cp -a /usr/local/lib/node_modules/. /atom-staged/lib/node_modules/; \
+    ln -sfn /atom-root/opt/wrapper/nodejs/packages/atom /atom-root/opt/wrapper/nodejs/packages/atom-jar /atom-staged/lib/node_modules/@appthreat/; \
+    printf '#!/bin/sh\nexport LD_LIBRARY_PATH=/atom-root/lib\nexport JAVA_HOME=/atom-root%s\nexport PATH=/atom-root%s/bin:/atom-root%s/bin:/atom-root%s/bin:/atom-root%s/bin:/atom-root/opt/vendor/bin:$PATH\nexec /atom-root/bin/node /atom-root/opt/wrapper/nodejs/packages/atom/index.js "$@"\n' \
+      "$JAVA_DIR" "$JAVA_DIR" "$MAVEN_DIR" "$SBT_DIR" "$SCALA_DIR" \
+      > /atom-staged/bin/atom; \
+    chmod +x /atom-staged/bin/atom
+
+ENTRYPOINT ["/bin/sh", "-c", "set -e; find /atom-root -mindepth 1 -delete; cp -a /atom-staged/. /atom-root/"]

docker-compose.yml

@@ -48,6 +48,7 @@ services:
       - workspace:/var/scancodeio/workspace/
       - static:/var/scancodeio/static/
       - healthycode_venv:/opt/healthycode
+      - atom_root:/atom-root
     restart: always
     depends_on:
       db:
@@ -69,6 +70,7 @@ services:
       - /etc/scancodeio/:/etc/scancodeio/
       - workspace:/var/scancodeio/workspace/
       - healthycode_venv:/opt/healthycode
+      - atom_root:/atom-root
     restart: always
     depends_on:
       db:
@@ -107,11 +109,20 @@ services:
       - healthycode_venv:/opt/healthycode
     restart: "no"
 
+  atom:
+    profiles: [atom]
+    build:
+      context: .
+      dockerfile: atom.Dockerfile
+    volumes: [atom_root:/atom-root]
+    restart: "no"
+
 volumes:
   db_data:
   redis_data:
   clamav_data:
   static:
   workspace:
   webroot:
-  healthycode_venv:
+  healthycode_venv:
+  atom_root:

docker.env

@@ -7,4 +7,5 @@ SCANCODEIO_DB_HOST=db
 SCANCODEIO_RQ_REDIS_HOST=redis
 SCANCODEIO_ASYNC=True
 SCANCODEIO_WORKSPACE_LOCATION=/var/scancodeio/workspace/
-GRIMOIRELAB_METRICS_EXECUTABLE=/opt/healthycode/.venv/bin/grimoirelab-metrics
+GRIMOIRELAB_METRICS_EXECUTABLE=/opt/healthycode/.venv/bin/grimoirelab-metrics
+ATOM_EXECUTABLE=/atom-root/bin/atom

docs/index.rst

@@ -48,6 +48,7 @@ Learn via practical step-by-step guides.
 - :ref:`tutorial_web_ui_symbol_and_string_collection`
 - :ref:`tutorial_cli_end_to_end_scanning_to_dejacode`
 - :ref:`tutorial_analyze_symbols_reachability`
+- :ref:`tutorial_atom_reachability_integration_tools`
 - :ref:`tutorial_reachability_integration_tools`
 - :ref:`tutorial_web_ui_scan_repo_health`
 
@@ -121,6 +122,7 @@ Indices and tables
     tutorial_cli_end_to_end_scanning_to_dejacode
     tutorial_analyze_symbols_reachability
     tutorial_reachability_integration_tools
+    tutorial_atom_reachability_integration_tools
     tutorial_web_ui_scan_repo_health
     scanpipe-concepts
     built-in-pipelines

docs/tutorial_atom_reachability_integration_tools.rst

@@ -0,0 +1,115 @@
+.. _tutorial_atom_reachability_integration_tools:
+
+Atom Reachability Tools
+========================
+
+The ``atom_reachability`` pipeline uses
+`Atom <https://github.com/AppThreat/atom>`_ to determine whether code symbols
+referenced by a vulnerability patch are reachable. Atom is run twice against
+the project codebase:
+
+- ``usages`` extracts local variable and parameter usages from the code and
+  writes a ``.atom`` file plus a ``usages`` slice.
+- ``reachables`` reads that ``.atom`` file and writes a ``reachables`` slice
+  of the symbols that can actually be reached, based on automatically
+  generated framework tags.
+
+Each advisory is classified as ``yes``, ``unknown``, or ``no``.
+
+Requirements
+------------
+
+- `Atom tool <https://github.com/AppThreat/atom>`_ installed and configured.
+
+Configuration
+-------------
+
+Set this environment variable in your ``.env``:
+
+.. code-block:: bash
+
+    ATOM_EXECUTABLE="/home/user/.npm-global/bin/atom"
+
+Running the pipeline
+--------------------
+
+1. Open a Project that contains resources and at least one package
+   vulnerability.
+2. Click **"Add pipeline"** and select **"atom_reachability"**.
+3. Select **"Execute pipeline now"** and click **"Add pipeline"**.
+4. After completion, open the **Resources** view (click the count under
+   **"RESOURCES"**), then click an affected file and open the **Extra**
+   tab to view the ``symbols_reachability`` data.
+
+.. image:: images/tutorial-web-ui-atom-symbol-reachability.png
+
+Output
+------
+
+The pipeline writes four timestamped files three JSON documents and one
+binary atom file and embeds the same advisory data in each resource's
+``extra_data.symbols_reachability``.
+
+1. ``reachability-YYYY-MM-DD-HH-MM-SS.json`` is the advisory reachability
+   report. It aggregates results per advisory, listing each advisory with
+   its overall verdict, matched vulnerable/fixed symbols, and per-symbol
+   witness paths.
+   Schema:
+
+.. code-block:: json
+    :emphasize-lines: 2, 7, 9, 13
+
+    {
+      "purl": "pkg:pypi/b2sdk@1.14.0",
+      "advisories": [
+        {
+          "advisory_uid": "pypa/b2sdk/PYSEC-2022-33",
+          "is_reachable": "yes",
+          "details": [
+            {
+              "symbol_name": "SqliteAccountInfo._create_database",
+              "in_usage": true,
+              "calls": [],
+              "in_reachable": true
+            }
+          ],
+          "vulnerable_symbols": [
+            "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo",
+            "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo._create_database",
+          ],
+          "fixed_symbols": [
+            "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo",
+            "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo._create_database",
+          ]
+        }
+      ]
+    }
+
+2. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.usages.slices.json`` is the
+   ``usages`` slice generated by atom.
+
+.. code-block:: text
+
+    {
+      "objectSlices": [...],
+      "userDefinedTypes": [...]
+    }
+
+3. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.reachables.slices.json`` is
+   the ``reachables`` slice generated by atom.
+
+.. code-block:: text
+
+    [
+      {
+        "flows": [
+        ...
+        ],
+        "purls": []
+      },
+      ...
+    ]
+
+4. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.atom`` is atom's binary
+   CPG. It can be re-sliced, exported to other tools,
+   and queried with graph algorithms using the atom CLI.

pyproject.toml

@@ -156,6 +156,7 @@ analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker"
 analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS"
 analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows"
 analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability"
+analyze_atom_reachability = "scanpipe.pipelines.analyze_atom_reachability:AtomReachability"
 benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls"
 collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext"
 collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags"

scanpipe/pipelines/analyze_atom_reachability.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from scanpipe.pipes import atom
+from scanpipe.pipes.reachability_tools import ReachabilityPipeline
+
+
+class AtomReachability(ReachabilityPipeline):
+    """
+    Determine the reachability of vulnerabilities using the atom tool.
+
+    Note: You must run the ``find_vulnerabilities`` pipeline before running
+    this pipeline.
+
+    atom is run twice against the project codebase:
+
+    - ``usages`` writes a usages slice of the code.
+    - ``reachables`` writes a reachables slice of the symbols
+     that can actually be reached.
+
+    The vulnerable and fixed symbols of each patch are matched against those
+    slices:
+
+    - REACHABLE: the symbol has a call site in the code
+    - UNKNOWN: the symbol is defined in the code
+    - NOT_REACHABLE: the symbol is absent
+
+    Results are stored in the ``extra_data`` of the matching resources under
+    the ``symbols_reachability`` key.
+    """
+
+    reachability_tool = atom.AtomTool
+
+    @classmethod
+    def get_availability(cls):
+        return cls.reachability_tool.get_availability()
+
+    def get_candidate_resources(self):
+        """Collect candidate resources for this pipeline."""
+        self.candidate_resources = self.project.codebaseresources.files().filter(
+            is_binary=False,
+            is_archive=False,
+            is_media=False,
+            programming_language__in=list(self.reachability_tool.supported_languages),
+        )
+
+    def collect_resource_index(self):
+        """Run the atom usages and reachables commands and load both slices."""
+        self.resource_indexe_paths = self.reachability_tool.run(
+            project=self.project, logger=self.log
+        )
+        indexes = self.reachability_tool.parsed_output(
+            export_output_path=self.resource_indexe_paths, logger=self.log
+        )
+        self.resource_indexes_usages = indexes.get("usages") or {}
+        self.resource_indexes_reachability = indexes.get("reachables") or {}
+
+    def collect_and_match_resources(self):
+        """Match patch symbols against the atom slices."""
+        if not self.resource_indexes_usages and not self.resource_indexes_reachability:
+            self.log("No atom usages or reachables slices available for matching.")
+            return
+
+        atom.collect_and_match_resources(
+            tool_name=self.reachability_tool.tool_name,
+            patches=self.patches,
+            patch_symbols=self.patch_symbols,
+            usages_slices=self.resource_indexes_usages,
+            reachables_slices=self.resource_indexes_reachability,
+            candidate_resources=self.candidate_resources,
+            logger=self.log,
+        )