diff --git a/README.rst b/README.rst index 11a4dfb..53b3976 100644 --- a/README.rst +++ b/README.rst @@ -1,107 +1,49 @@ -A Simple Python Project Skeleton -================================ +================= +ReachableCode +================= -This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. -Using this `blog post`_ as inspiration, this repository serves as the base for -all new Python projects and is mergeable in existing repositories as well. +ReachableCode is a collection of pipelines for analyzing code reachability. +It integrates with ScanCode.io as a pipeline plugin +and uses tools like (Atom, CPG, .. ) to determine whether +vulnerable code symbols are reachable within a project's codebase. -.. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ +Other information: +- License: Apache-2.0 +- Copyright (c) nexB Inc., AboutCode.org and others. +- Homepage: https://github.com/aboutcode-org/reachablecode/ ++ Documentation: https://reachablecode.readthedocs.io/ -Usage -===== +Requirements +------------ -A brand new project -------------------- +- ScanCode.io https://github.com/nexB/scancode.io -.. code-block:: bash +See also the next section for detailed instructions. - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git +Local installation and usage test +------------------------------------ -From here, you can make the appropriate changes to the files for your specific project. +To install: +~~~~~~~~~~~~ -Update an existing project ---------------------------- +- Clone reachablecode locally side-by-side: ``git clone https://github.com/aboutcode-org/reachablecode`` +- Clone ScanCode.io locally side-by-side: ``git clone https://github.com/aboutcode-org/scancode.io`` +- Change to the scancode.io directory and run ``make dev`` then ``source bin/activate`` -.. code-block:: bash + - Follow the full instructions at https://scancodeio.readthedocs.io/en/latest/installation.html#local-development-installation - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories +- Run ``pip install --editable ../reachablecode`` +- Run ScanCode.io with ``./manage.py runserver --insecure`` and open the URL in your browser. + There is a new "analyze_atom_reachability" pipeline available when creating a new project. -This is also the workflow to use when updating the skeleton files in any given repository. +Development +---------------- -More usage instructions can be found in ``docs/skeleton-usage.rst``. +- Install requirements and dependencies using ``make dev`` +- Then ``source venv/bin/activate`` +Testing: -Release Notes -============= - -- 2025-03-31: - - - Use ruff as the main code formatting tool, add ruff rules to pyproject.toml - -- 2025-03-29: - - - Add support for beta macOS-15 - - Add support for beta windows-2025 - -- 2025-02-14: - - - Drop support for Python 3.8, add support in CI for Python 3.13, use Python 3.12 as default - version. - -- 2025-01-17: - - - Drop support for macOS-12, add support for macOS-14 - - Add support in CI for ubuntu-24.04 - - Add support in CI for Python 3.12 - -- 2024-08-20: - - - Update references of ownership from nexB to aboutcode-org - -- 2024-07-01: - - - Drop support for Python 3.8 - - Drop support for macOS-11, add support for macOS-14 - -- 2024-02-19: - - - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 - -- 2023-10-18: - - - Add dark mode support in documentation - -- 2023-07-18: - - - Add macOS-13 job in azure-pipelines.yml - -- 2022-03-04: - - - Synchronize configure and configure.bat scripts for sanity - - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party - dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - -- 2021-09-03: - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` - and ``requirements-dev.txt`` - - ``configure`` can now accept multiple options at once - - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` - and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating - ABOUT files for them. - -- 2021-05-11: - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which - Python version is used. +- To run tests: ``pytest -vvs`` diff --git a/atom.Dockerfile b/atom.Dockerfile new file mode 100644 index 0000000..b9df9a7 --- /dev/null +++ b/atom.Dockerfile @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +FROM ghcr.io/appthreat/atom:v3.2.0 + +RUN set -e; \ + JAVA_DIR="$(ls -d /opt/java/*-tem | head -n1)"; \ + MAVEN_DIR="$(ls -d /opt/maven/* | head -n1)"; \ + SBT_DIR="$(ls -d /opt/sbt/* | head -n1)"; \ + SCALA_DIR="$(ls -d /opt/scala/* | head -n1)"; \ + mkdir -p /atom-staged/opt /atom-staged/bin /atom-staged/lib/node_modules; \ + cp -a /opt/wrapper /opt/java /opt/maven /opt/sbt /opt/scala /opt/vendor /atom-staged/opt/; \ + cp /usr/bin/node-22 /atom-staged/bin/node; \ + for lib in $(ldd /usr/bin/node-22 | awk '/=>/{print $3}'); do \ + case "$lib" in */libc.so*|*/libm.so*|*/libpthread*|*/libdl.so*|*/librt*|*/ld-linux*) ;; *) cp -aL "$lib" /atom-staged/lib/ ;; esac; \ + done; \ + cp -a /usr/local/lib/node_modules/. /atom-staged/lib/node_modules/; \ + ln -sfn /atom-root/opt/wrapper/nodejs/packages/atom /atom-root/opt/wrapper/nodejs/packages/atom-jar /atom-staged/lib/node_modules/@appthreat/; \ + printf '#!/bin/sh\nexport LD_LIBRARY_PATH=/atom-root/lib\nexport JAVA_HOME=/atom-root%s\nexport PATH=/atom-root%s/bin:/atom-root%s/bin:/atom-root%s/bin:/atom-root%s/bin:/atom-root/opt/vendor/bin:$PATH\nexec /atom-root/bin/node /atom-root/opt/wrapper/nodejs/packages/atom/index.js "$@"\n' \ + "$JAVA_DIR" "$JAVA_DIR" "$MAVEN_DIR" "$SBT_DIR" "$SCALA_DIR" \ + > /atom-staged/bin/atom; \ + chmod +x /atom-staged/bin/atom + +ENTRYPOINT ["/bin/sh", "-c", "set -e; find /atom-root -mindepth 1 -delete; cp -a /atom-staged/. /atom-root/"] \ No newline at end of file diff --git a/configure b/configure index 6d317d4..e111d23 100755 --- a/configure +++ b/configure @@ -28,14 +28,15 @@ CLI_ARGS=$1 ################################ # Requirement arguments passed to pip and used by default or with --dev. -REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" +REQUIREMENTS="--editable ." +DEV_REQUIREMENTS="--editable .[testing,docs]" +DOCS_REQUIREMENTS="--editable .[docs]" # where we create a virtualenv VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" +CLEANABLE="build dist venv .cache .eggs" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -110,7 +111,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --pip embed --setuptools embed \ + --wheel embed --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ @@ -167,7 +168,6 @@ clean() { for cln in $CLEANABLE; do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; done - find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete set +e exit } @@ -185,6 +185,7 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b2bbe2d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,131 @@ +name: scancodeio + +x-description: "Full ScanCode.io stack: all services, local image build" + +services: + db: + image: docker.io/library/postgres:17.11 + env_file: + - docker.env + volumes: + - db_data:/var/lib/postgresql/data/ + shm_size: "1gb" + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: docker.io/library/redis:8.10.2-alpine + # Enable redis data persistence using the "Append Only File" with the + # default policy of fsync every second. See https://redis.io/topics/persistence + command: redis-server --appendonly yes --loglevel warning + volumes: + - redis_data:/data + restart: always + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + web: + build: + context: ../scancode.io + command: sh -c " + ./manage.py migrate && + ./manage.py collectstatic --no-input --verbosity 0 --clear && + gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 \ + --workers ${GUNICORN_WORKERS:-8} --worker-tmp-dir /dev/shm" + env_file: + - docker.env + expose: + - 8000 + volumes: + - .env:/opt/scancodeio/.env + - /etc/scancodeio/:/etc/scancodeio/ + - workspace:/var/scancodeio/workspace/ + - static:/var/scancodeio/static/ + - healthycode_venv:/opt/healthycode + - atom_root:/atom-root + restart: always + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + worker: + build: + context: ../scancode.io + # Ensure that potential db migrations run first by waiting until "web" is up + command: wait-for-it --strict --timeout=600 web:8000 -- sh -c " + ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker + --queue-class scancodeio.worker.ScanCodeIOQueue + --verbosity 1" + env_file: + - docker.env + volumes: + - .env:/opt/scancodeio/.env + - /etc/scancodeio/:/etc/scancodeio/ + - workspace:/var/scancodeio/workspace/ + - healthycode_venv:/opt/healthycode + - atom_root:/atom-root + restart: always + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + web: + condition: service_started + + nginx: + image: docker.io/library/nginx:1.31.6-alpine + ports: + - "${NGINX_PUBLISHED_HTTP_PORT:-80}:80" + - "${NGINX_PUBLISHED_HTTPS_PORT:-443}:443" + volumes: + - ./etc/nginx/conf.d/:/etc/nginx/conf.d/ + - static:/var/scancodeio/static/ + - webroot:/var/www/html/ + depends_on: + web: + condition: service_started + restart: always + + clamav: + image: docker.io/clamav/clamav:1.5.4_base + volumes: + - clamav_data:/var/lib/clamav + - workspace:/var/scancodeio/workspace/ + restart: always + + healthycode: + profiles: + - healthycode + image: ghcr.io/aboutcode-org/healthycode:0.2.0 + volumes: + - healthycode_venv:/opt/healthycode + restart: "no" + + atom: + profiles: + - atom + build: + context: . + dockerfile: atom.Dockerfile + volumes: [atom_root:/atom-root] + restart: "no" + +volumes: + db_data: + redis_data: + clamav_data: + static: + workspace: + webroot: + healthycode_venv: + atom_root: \ No newline at end of file diff --git a/docker.env b/docker.env new file mode 100644 index 0000000..48d19ef --- /dev/null +++ b/docker.env @@ -0,0 +1,10 @@ +POSTGRES_DB=scancodeio +POSTGRES_USER=scancodeio +POSTGRES_PASSWORD=scancodeio +POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 + +SCANCODEIO_DB_HOST=db +SCANCODEIO_RQ_REDIS_HOST=redis +SCANCODEIO_ASYNC=True +SCANCODEIO_WORKSPACE_LOCATION=/var/scancodeio/workspace/ +ATOM_EXECUTABLE=/atom-root/bin/atom \ No newline at end of file diff --git a/docs/source/images/tutorial-web-ui-atom-symbol-reachability.png b/docs/source/images/tutorial-web-ui-atom-symbol-reachability.png new file mode 100644 index 0000000..3cd9119 Binary files /dev/null and b/docs/source/images/tutorial-web-ui-atom-symbol-reachability.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index eb63717..42b479a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,7 @@ Welcome to nexb-skeleton's documentation! :maxdepth: 2 :caption: Contents: - skeleton-usage + tutorial_atom_reachability_integration_tools contribute/contrib_doc Indices and tables diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst deleted file mode 100644 index 6cb4cc5..0000000 --- a/docs/source/skeleton-usage.rst +++ /dev/null @@ -1,160 +0,0 @@ -Usage -===== -A brand new project -------------------- -.. code-block:: bash - - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton - - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git - -From here, you can make the appropriate changes to the files for your specific project. - -Update an existing project ---------------------------- -.. code-block:: bash - - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories - -This is also the workflow to use when updating the skeleton files in any given repository. - -Customizing ------------ - -You typically want to perform these customizations: - -- remove or update the src/README.rst and tests/README.rst files -- set project info and dependencies in setup.cfg -- check the configure and configure.bat defaults - -Initializing a project ----------------------- - -All projects using the skeleton will be expected to pull all of it dependencies -from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using -requirements.txt and/or requirements-dev.txt to determine what version of a -package to collect. By default, PyPI will not be used to find and collect -packages from. - -In the case where we are starting a new project where we do not have -requirements.txt and requirements-dev.txt and whose dependencies are not yet on -thirdparty.aboutcode.org/pypi, we run the following command after adding and -customizing the skeleton files to your project: - -.. code-block:: bash - - ./configure - -This will initialize the virtual environment for the project, pull in the -dependencies from PyPI and add them to the virtual environment. - - -Generating requirements.txt and requirements-dev.txt ----------------------------------------------------- - -After the project has been initialized, we can generate the requirements.txt and -requirements-dev.txt files. - -Ensure the virtual environment is enabled. - -.. code-block:: bash - - source venv/bin/activate - -To generate requirements.txt: - -.. code-block:: bash - - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ - -Replace \ with the version number of the Python being used, for example: -``venv/lib/python3.6/site-packages/`` - -To generate requirements-dev.txt after requirements.txt has been generated: - -.. code-block:: bash - - ./configure --dev - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ - -Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` - -.. code-block:: bash - - python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --dev - python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ - - -Collecting and generating ABOUT files for dependencies ------------------------------------------------------- - -Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: - -.. code-block:: bash - - pip install -r etc/scripts/requirements.txt - -Once we have requirements.txt and requirements-dev.txt, we can fetch the project -dependencies as wheels and generate ABOUT files for them: - -.. code-block:: bash - - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt - -There may be issues with the generated ABOUT files, which will have to be -corrected. You can check to see if your corrections are valid by running: - -.. code-block:: bash - - python etc/scripts/check_thirdparty.py -d thirdparty - -Once the wheels are collected and the ABOUT files are generated and correct, -upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT -files from the thirdparty directory to the pypi directory at -https://github.com/aboutcode-org/thirdparty-packages - - -Usage after project initialization ----------------------------------- - -Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated -and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically -when you update dependencies or use a new checkout. - -If the virtual env for the project becomes polluted, or you would like to remove -it, use the ``--clean`` option: - -.. code-block:: bash - - ./configure --clean - -Then you can run ``./configure`` again to set up the project virtual environment. - -To set up the project for development use: - -.. code-block:: bash - - ./configure --dev - -To update the project dependencies (adding, removing, updating packages, etc.), -update the dependencies in ``setup.cfg``, then run: - -.. code-block:: bash - - ./configure --clean # Remove existing virtual environment - source venv/bin/activate # Ensure virtual environment is activated - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt - pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files - -Ensure that the generated ABOUT files are valid, then take the dependency wheels -and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/docs/source/tutorial_atom_reachability_integration_tools.rst b/docs/source/tutorial_atom_reachability_integration_tools.rst new file mode 100644 index 0000000..a6cd99e --- /dev/null +++ b/docs/source/tutorial_atom_reachability_integration_tools.rst @@ -0,0 +1,115 @@ +.. _tutorial_atom_reachability_integration_tools: + +Atom Reachability Tools +======================== + +The ``atom_reachability`` pipeline uses +`Atom `_ to determine whether code symbols +referenced by a vulnerability patch are reachable. Atom is run twice against +the project codebase: + +- ``usages`` extracts local variable and parameter usages from the code and + writes a ``.atom`` file plus a ``usages`` slice. +- ``reachables`` reads that ``.atom`` file and writes a ``reachables`` slice + of the symbols that can actually be reached, based on automatically + generated framework tags. + +Each advisory is classified as ``yes``, ``unknown``, or ``no``. + +Requirements +------------ + +- `Atom tool `_ installed and configured. + +Configuration +------------- + +Set this environment variable in your ``.env``: + +.. code-block:: bash + + ATOM_EXECUTABLE="/home/user/.npm-global/bin/atom" + +Running the pipeline +-------------------- + +1. Open a Project that contains resources and at least one package + vulnerability. +2. Click **"Add pipeline"** and select **"atom_reachability"**. +3. Select **"Execute pipeline now"** and click **"Add pipeline"**. +4. After completion, open the **Resources** view (click the count under + **"RESOURCES"**), then click an affected file and open the **Extra** + tab to view the ``symbols_reachability`` data. + +.. image:: images/tutorial-web-ui-atom-symbol-reachability.png + +Output +------ + +The pipeline writes four timestamped files three JSON documents and one +binary atom file and embeds the same advisory data in each resource's +``extra_data.symbols_reachability``. + +1. ``reachability-YYYY-MM-DD-HH-MM-SS.json`` is the advisory reachability + report. It aggregates results per advisory, listing each advisory with + its overall verdict, matched vulnerable/fixed symbols, and per-symbol + witness paths. + Schema: + +.. code-block:: json + :emphasize-lines: 2, 7, 9, 13 + + { + "purl": "pkg:pypi/b2sdk@1.14.0", + "advisories": [ + { + "advisory_uid": "pypa/b2sdk/PYSEC-2022-33", + "is_reachable": "yes", + "details": [ + { + "symbol_name": "SqliteAccountInfo._create_database", + "in_usage": true, + "calls": [], + "in_reachable": true + } + ], + "vulnerable_symbols": [ + "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo", + "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo._create_database", + ], + "fixed_symbols": [ + "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo", + "b2sdk/account_info/sqlite_account_info.py::SqliteAccountInfo._create_database", + ] + } + ] + } + +2. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.usages.slices.json`` is the + ``usages`` slice generated by atom. + +.. code-block:: text + + { + "objectSlices": [...], + "userDefinedTypes": [...] + } + +3. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.reachables.slices.json`` is + the ``reachables`` slice generated by atom. + +.. code-block:: text + + [ + { + "flows": [ + ... + ], + "purls": [] + }, + ... + ] + +4. ``atom_reachability-YYYY-MM-DD-HH-MM-SS.python.atom`` is atom's binary + CPG. It can be re-sliced, exported to other tools, + and queried with graph algorithms using the atom CLI. diff --git a/pyproject.toml b/pyproject.toml index f106e69..ed5c8f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,6 @@ requires = ["setuptools >= 50", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools_scm] -# this is used populated when creating a git archive -# and when there is .git dir and/or there is no git installed -fallback_version = "9999.$Format:%h-%cs$" - [tool.pytest.ini_options] norecursedirs = [ ".git", @@ -68,7 +63,7 @@ include = [ "." ] -# ignore test data and testfiles: they should never be linted nor formatted +# ignore test data and testfiles: they should never be linted nor formatted exclude = [ # main style "**/tests/data/**/*", @@ -126,4 +121,4 @@ max-complexity = 10 [tool.doc8] ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] -max-line-length=100 +max-line-length=100 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a0f2985..443a4c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,14 +1,15 @@ [metadata] -name = skeleton +name = reachablecode +version = 0.0.1 license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 -description = skeleton +description = A collection of ScanCode.io pipelines dedicated to reachability analysis. long_description = file:README.rst long_description_content_type = text/x-rst -url = https://github.com/aboutcode-org/skeleton +url = https://github.com/aboutcode-org/reachablecode -author = nexB. Inc. and others +author = nexB. Inc., AboutCode and others author_email = info@aboutcode.org classifiers = @@ -31,15 +32,18 @@ license_files = README.rst [options] -python_requires = >=3.10 - package_dir = =src packages = find: include_package_data = true zip_safe = false +setup_requires = setuptools_scm[toml] >= 4 + +python_requires = >=3.8 + install_requires = + commoncode [options.packages.find] @@ -47,12 +51,18 @@ where = src [options.extras_require] -dev = - pytest >= 7.0.1 +testing = + pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 + pycodestyle >= 2.8.0 twine - ruff + black + isort + scancodeio + pytest-django + +docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 @@ -60,3 +70,8 @@ dev = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton + + +[options.entry_points] +scancodeio_pipelines = + analyze_atom_reachability = reachablecode.pipelines.analyze_atom_reachability:AtomReachability \ No newline at end of file diff --git a/src/README.rst b/src/README.rst deleted file mode 100644 index ec651fc..0000000 --- a/src/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python source code (and installable data) in this directory. - diff --git a/src/reachablecode/__init__.py b/src/reachablecode/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/reachablecode/pipelines/__init__.py b/src/reachablecode/pipelines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/reachablecode/pipelines/analyze_atom_reachability.py b/src/reachablecode/pipelines/analyze_atom_reachability.py new file mode 100644 index 0000000..aacfc9e --- /dev/null +++ b/src/reachablecode/pipelines/analyze_atom_reachability.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from scanpipe.pipes.reachability_tools import ReachabilityPipeline + +from reachablecode.pipes import atom + + +class AtomReachability(ReachabilityPipeline): + """ + Determine the reachability of vulnerabilities using the atom tool. + + Note: You must run the ``find_vulnerabilities`` pipeline before running + this pipeline. + + atom is run twice against the project codebase: + + - ``usages`` writes a usages slice of the code. + - ``reachables`` writes a reachables slice of the symbols + that can actually be reached. + + The vulnerable and fixed symbols of each patch are matched against those + slices: + + - REACHABLE: the symbol has a call site in the code + - UNKNOWN: the symbol is defined in the code + - NOT_REACHABLE: the symbol is absent + + Results are stored in the ``extra_data`` of the matching resources under + the ``symbols_reachability`` key. + """ + + reachability_tool = atom.AtomTool + + @classmethod + def get_availability(cls): + return cls.reachability_tool.get_availability() + + def get_candidate_resources(self): + """Collect candidate resources for this pipeline.""" + self.candidate_resources = self.project.codebaseresources.files().filter( + is_binary=False, + is_archive=False, + is_media=False, + programming_language__in=list(self.reachability_tool.supported_languages), + ) + + def collect_resource_index(self): + """Run the atom usages and reachables commands and load both slices.""" + self.resource_indexe_paths = self.reachability_tool.run( + project=self.project, logger=self.log + ) + indexes = self.reachability_tool.parsed_output( + export_output_path=self.resource_indexe_paths, logger=self.log + ) + self.resource_indexes_usages = indexes.get("usages") or {} + self.resource_indexes_reachability = indexes.get("reachables") or {} + + def collect_and_match_resources(self): + """Match patch symbols against the atom slices.""" + if not self.resource_indexes_usages and not self.resource_indexes_reachability: + self.log("No atom usages or reachables slices available for matching.") + return + + atom.collect_and_match_resources( + tool_name=self.reachability_tool.tool_name, + patches=self.patches, + patch_symbols=self.patch_symbols, + usages_slices=self.resource_indexes_usages, + reachables_slices=self.resource_indexes_reachability, + candidate_resources=self.candidate_resources, + logger=self.log, + ) \ No newline at end of file diff --git a/src/reachablecode/pipes/__init__.py b/src/reachablecode/pipes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/reachablecode/pipes/atom.py b/src/reachablecode/pipes/atom.py new file mode 100644 index 0000000..1093724 --- /dev/null +++ b/src/reachablecode/pipes/atom.py @@ -0,0 +1,560 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json +import re +from collections import defaultdict +from os import environ +from subprocess import SubprocessError + +from aboutcode.pipeline import LoopProgress +from scanpipe.pipes import run_command_safely +from scanpipe.pipes.reachability import ReachabilityStatus, save_resource_reachability_report +from scanpipe.pipes.reachability_tools import ReachabilityTool + +ATOM_EXECUTABLE = environ.get("ATOM_EXECUTABLE") + +_OPERATOR_PREFIXES = ("", "") +_MARKER_PATTERN = re.compile(r"<[^>]+>") + + +def normalize_file_path(path): + """Return ``path`` as a normalized forward-slash relative path.""" + path = (path or "").replace("\\", "/") + while path.startswith("./"): + path = path[2:] + return path.lstrip("/") + + +def normalize_symbol_name(name): + """Convert an atom name to a comparable dotted form, or ``""`` if unusable.""" + if not isinstance(name, str): + return "" + name = name.strip() + if not name or name.startswith(_OPERATOR_PREFIXES): + return "" + + if "::" in name: + name = name.partition("::")[2] + + if ":" in name: + # Convert ``pkg/mod.py:.Class.method`` to ``pkg.mod.Class.method``. + path, _, tail = name.partition(":") + path = normalize_file_path(path).removesuffix(".py") + name = path.replace("/", ".") + "." + tail + + name = _MARKER_PATTERN.sub("", name) + + while ".." in name: + name = name.replace("..", ".") + + return name.strip(".") + + +class SliceMatcher: + """Index the dotted symbol names of an atom slice, grouped by file.""" + + def __init__(self, slices): + self.by_file = defaultdict(set) + self.by_basename = defaultdict(set) + if slices: + self.parse_slice(slices) + + @property + def file_count(self): + """Return the number of files indexed in this slice.""" + return len(self.by_file) + + def parse_slice(self, slices): + """Populate the indexes from ``slices``. Subclasses must implement this.""" + raise NotImplementedError + + def add_symbols(self, file_name, *names): + """Index each symbol in ``names`` under ``file_name``, in dotted form.""" + if not isinstance(file_name, str) or not file_name: + return + + path = normalize_file_path(file_name) + basename = path.rsplit("/", 1)[-1] + for name in names: + dotted = normalize_symbol_name(name) + if dotted: + self.by_file[path].add(dotted) + self.by_basename[basename].add(dotted) + + def match_symbols(self, symbols, file_name): + """Return the subset of ``symbols`` present in the slice for ``file_name``.""" + if not symbols: + return set() + + path = normalize_file_path(file_name) + names = self.by_file.get(path) or self.by_basename.get(path.rsplit("/", 1)[-1]) + if not names: + return set() + + matched = set() + for symbol in symbols: + dotted = normalize_symbol_name(symbol) + if not dotted: + continue + + if any( + name == dotted + or name.endswith("." + dotted) + or ("." in name and dotted.endswith("." + name)) + for name in names + ): + matched.add(symbol) + + return matched + + +class UsagesMatcher(SliceMatcher): + """Index the usages slice (objectSlices / userDefinedTypes) and its calls.""" + + def __init__(self, slices): + self.calls_by_file = defaultdict(dict) + self.calls_by_basename = defaultdict(dict) + super().__init__(slices) + + def parse_slice(self, slices): + if not isinstance(slices, dict): + return + + for obj in slices.get("objectSlices") or []: + self.add_object_slice(obj or {}) + + for typ in slices.get("userDefinedTypes") or []: + self.add_user_defined_type(typ or {}) + + def add_calls(self, file_name, symbols, calls): + """Associate ``calls`` with each symbol in ``symbols`` for ``file_name``.""" + if not isinstance(file_name, str) or not file_name or not calls: + return + + path = normalize_file_path(file_name) + basename = path.rsplit("/", 1)[-1] + for symbol in symbols: + dotted = normalize_symbol_name(symbol) + if not dotted: + continue + + file_slot = self.calls_by_file[path].setdefault(dotted, []) + base_slot = self.calls_by_basename[basename].setdefault(dotted, []) + + for call in calls: + if call not in file_slot: + file_slot.append(call) + if call not in base_slot: + base_slot.append(call) + + def get_calls(self, symbols, file_name): + """ + Return the calls associated with ``symbols`` in ``file_name``. + + Uses the same fuzzy matching as ``match_symbols`` (exact, suffix, or + prefix), deduplicated by ``(call_name, resolved_method, line_number)``. + """ + if not symbols: + return [] + + path = normalize_file_path(file_name) + file_calls = self.calls_by_file.get(path) + if not file_calls: + file_calls = self.calls_by_basename.get(path.rsplit("/", 1)[-1], {}) + if not file_calls: + return [] + + result = [] + seen = set() + for symbol in symbols: + dotted = normalize_symbol_name(symbol) + if not dotted: + continue + + for name, name_calls in file_calls.items(): + if not ( + name == dotted + or name.endswith("." + dotted) + or ("." in name and dotted.endswith("." + name)) + ): + continue + + for call in name_calls: + key = ( + call.get("call_name"), + call.get("resolved_method"), + call.get("line_number"), + ) + if key not in seen: + seen.add(key) + result.append(call) + return result + + def build_call_record(self, call): + """Return a compact call record from an atom call entry, or ``None``.""" + call = call or {} + call_name = call.get("callName") + resolved = call.get("resolvedMethod") + if not (call_name or resolved): + return None + + record = {} + if call_name: + record["call_name"] = call_name + + if resolved: + record["resolved_method"] = resolved + + if call.get("lineNumber") is not None: + record["line_number"] = call.get("lineNumber") + + if call.get("columnNumber") is not None: + record["column_number"] = call.get("columnNumber") + + return record + + def add_object_slice(self, obj): + """Index the symbols and calls of one ``objectSlices`` entry.""" + file_name = obj.get("fileName") + self.add_symbols(file_name, obj.get("fullName")) + + for usage in obj.get("usages") or []: + usage = usage or {} + + calls = [] + for raw_call in (usage.get("argToCalls") or []) + ( + usage.get("invokedCalls") or [] + ): + raw_call = raw_call or {} + self.add_symbols( + file_name, + raw_call.get("callName"), + raw_call.get("resolvedMethod"), + ) + record = self.build_call_record(raw_call) + if record: + calls.append(record) + + usage_symbols = [] + for part in (usage.get("targetObj"), usage.get("definedBy")): + part = part or {} + names = ( + part.get("name"), + part.get("typeFullName"), + part.get("resolvedMethod"), + ) + self.add_symbols(file_name, *names) + usage_symbols.extend(name for name in names if name) + + if usage_symbols and calls: + self.add_calls(file_name, usage_symbols, calls) + + def add_user_defined_type(self, typ): + """Index the symbols of one ``userDefinedTypes`` entry.""" + file_name = typ.get("fileName") + self.add_symbols(file_name, typ.get("name")) + + for field in typ.get("fields") or []: + field = field or {} + self.add_symbols( + file_name, + field.get("name"), + field.get("typeFullName"), + field.get("resolvedMethod"), + ) + + for proc in typ.get("procedures") or []: + proc = proc or {} + self.add_symbols( + file_name, proc.get("callName"), proc.get("resolvedMethod") + ) + + +class ReachablesMatcher(SliceMatcher): + """Index the ``reachables`` slice (``{"reachables": [{flows, purls}]}``).""" + + def parse_slice(self, slices): + if not isinstance(slices, list): + return + + for entry in slices: + for flow in (entry or {}).get("flows") or []: + flow = flow or {} + self.add_symbols( + flow.get("parentFileName"), + flow.get("name"), + flow.get("fullName"), + flow.get("typeFullName"), + flow.get("parentMethodName"), + flow.get("parentClassName"), + self.compose_class_method_name(flow), + ) + + def compose_class_method_name(self, flow): + """Return ``Class.method`` from a reachables flow, or ``""``.""" + method = flow.get("parentMethodName") or "" + klass = flow.get("parentClassName") or "" + + if not method or not klass: + return "" + + if ":" in klass: + klass = klass.partition(":")[2] + + return f"{klass}.{method}" + + +def classify_reachability(in_code, reachable): + """ + Return the reachability verdict for a set of vulnerable symbols. + + reachable is the subset matched against the reachables slice. + in_code is the subset matched against the usages slice. + """ + if reachable: + return ReachabilityStatus.REACHABLE + + if in_code: + return ReachabilityStatus.UNKNOWN + + return ReachabilityStatus.NOT_REACHABLE + + +def collect_and_match_resources( + tool_name, + patches, + patch_symbols, + usages_slices, + reachables_slices, + candidate_resources, + logger=None, +): + """ + Match patch symbols against the atom slices and store a reachability report + on each matching candidate resource. + + - a vulnerable symbol in ``reachables`` → REACHABLE + - a vulnerable symbol only in ``usages`` → UNKNOWN + - a vulnerable symbol in neither → NOT_REACHABLE + """ + usages = UsagesMatcher(usages_slices) + reachables = ReachablesMatcher(reachables_slices) + + if logger: + logger( + f"atom match: {usages.file_count} files in usages, " + f"{reachables.file_count} files in reachables, " + f"{len(candidate_resources)} candidates, {len(patches)} patches" + ) + + progress = LoopProgress(len(patches), logger=logger) + for patch in progress.iter(patches): + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + advisory_uids = patch.get("advisory_uids") or [] + + patch_symbols_by_language = patch_symbols.get(commit_hash, {}) + if not patch_symbols_by_language: + continue + + for resource in candidate_resources: + lang_patch_symbols = patch_symbols_by_language.get( + resource.programming_language + ) + if not lang_patch_symbols: + continue + + vulnerable = lang_patch_symbols.get("vulnerable") or {} + fixed = lang_patch_symbols.get("fixed") or {} + + path = resource.path + vuln_in_code = usages.match_symbols(vulnerable, path) + vuln_reachable = reachables.match_symbols(vulnerable, path) + fixed_in_code = usages.match_symbols(fixed, path) + + if not (vuln_in_code or vuln_reachable or fixed_in_code): + continue + + reachability_status = classify_reachability(vuln_in_code, vuln_reachable) + tool_details = [] + for symbol in sorted(vuln_in_code | vuln_reachable): + tool_details.append( + { + "symbol_name": symbol, + "in_usage": symbol in vuln_in_code, + "calls": usages.get_calls([symbol], path), + "in_reachable": symbol in vuln_reachable, + } + ) + + report = { + "tool_name": tool_name, + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "advisory_uids": advisory_uids, + "tool_details": tool_details, + "fixed_symbols": sorted(fixed), + "vulnerable_symbols": sorted(vulnerable), + "is_reachable": reachability_status.value, + } + + save_resource_reachability_report( + resource=resource, + commit_hash=commit_hash, + vcs_url=vcs_url, + new_report=report, + ) + + +class AtomTool(ReachabilityTool): + tool_name = "atom" + executable = ATOM_EXECUTABLE + supported_languages = { + "Java": "java", + "Python": "python", + } + + commands = ("usages", "reachables") + + @classmethod + def get_availability(cls): + if not cls.executable: + return ( + "atom is not configured. Set the ATOM_EXECUTABLE environment variable." + ) + + @classmethod + def get_atom_language(cls, project): + """Return the atom ``-l`` value for ``project`` from the scan summary.""" + summary_path = project.get_latest_output("summary") + with open(summary_path) as f: + summary = json.load(f) + + primary_language = summary.get("primary_language") + if not primary_language: + raise ValueError( + f"No primary_language in the scan summary of {project.name!r}." + ) + + language = cls.supported_languages.get(primary_language) + if not language: + raise ValueError( + f"Unsupported primary_language {primary_language!r} for " + f"{project.name!r}. Supported: {', '.join(cls.supported_languages)}." + ) + return language + + @staticmethod + def get_output_paths(project, atom_language): + """Return the output paths shared by the atom commands.""" + manifest_path = project.get_output_file_path("atom_reachability", "json") + prefix = f"{manifest_path.stem}.{atom_language}" + return { + "atom": manifest_path.with_name(f"{prefix}.atom"), + "usages": manifest_path.with_name(f"{prefix}.usages.slices.json"), + "reachables": manifest_path.with_name(f"{prefix}.reachables.slices.json"), + } + + @classmethod + def run_command( + cls, command, target_path, atom_language, output_paths, logger=None + ): + """Run a single atom command and return its slices file path or None.""" + command_args = [ + cls.executable, + command, + "-l", + atom_language, + "--cache", + "none", + "-o", + str(output_paths["atom"]), + "-s", + str(output_paths[command]), + str(target_path), + ] + + if logger: + logger(f"Running atom {command} on {target_path} ...") + try: + run_command_safely(command_args=command_args) + except SubprocessError as error: + if logger: + logger(f"atom {command} failed: {error!r}") + return None + return output_paths[command] + + @classmethod + def run(cls, project, logger=None): + """ + Run the atom ``usages`` and ``reachables`` commands. + + Return a mapping of command name to its slices file path (or ``None`` + when that command failed). + """ + target_path = project.codebase_path + if not target_path: + raise ValueError( + f"Cannot run Atom tools against project {project.name!r}:" + f" codebase_path is None." + ) + atom_language = cls.get_atom_language(project) + output_paths = cls.get_output_paths( + project=project, atom_language=atom_language + ) + + results = {} + for command in cls.commands: + results[command] = cls.run_command( + command=command, + target_path=target_path, + atom_language=atom_language, + output_paths=output_paths, + logger=logger, + ) + return results + + @staticmethod + def load_slices(export_output_path, logger=None): + """Load and return a single atom slices file.""" + if not export_output_path: + return {} + try: + with open(export_output_path) as slice_file: + return json.load(slice_file) + except (FileNotFoundError, json.JSONDecodeError) as error: + if logger: + logger(f"Skipping atom slices {export_output_path}: {error!r}") + return {} + + @classmethod + def parsed_output(cls, export_output_path, logger=None): + """Load and return the atom slices keyed by command name.""" + if not export_output_path: + return {} + return { + command: cls.load_slices(export_output_path=path, logger=logger) + for command, path in export_output_path.items() + } \ No newline at end of file diff --git a/tests/README.rst b/tests/README.rst deleted file mode 100644 index d94783e..0000000 --- a/tests/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python test modules in this directory. - diff --git a/tests/data/atom_reachability/python/app.py b/tests/data/atom_reachability/python/app.py new file mode 100644 index 0000000..074c86a --- /dev/null +++ b/tests/data/atom_reachability/python/app.py @@ -0,0 +1,40 @@ +import os + +from flask import Flask +from flask import request + +app = Flask(__name__) + +debug = False + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +@app.route("/report") +def handle_request(): + """HTTP entry point so atom tags a framework-route source.""" + generator = ReportGenerator("/var/reports") + requested_file = request.args.get("file") + + # Helper function nested inside the route handler + def build_file_path(filename): + # VULNERABLE: Direct concatenation allows Path Traversal + return os.path.join(generator.base_dir, filename) + + if not requested_file: + return "Error: No file specified" + + target_path = build_file_path(requested_file) + + with open(target_path) as f: + return f.read() + + +def unrelated_top_level_function(): + """Test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/tests/data/atom_reachability/python/atom_reachability.python.reachables.slices.json b/tests/data/atom_reachability/python/atom_reachability.python.reachables.slices.json new file mode 100644 index 0000000..bcf2464 --- /dev/null +++ b/tests/data/atom_reachability/python/atom_reachability.python.reachables.slices.json @@ -0,0 +1,1418 @@ +[ + { + "flows": [ + { + "id": 147, + "label": "IDENTIFIER", + "name": "tmp0", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0 = request.args", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "framework-input" + }, + { + "id": 153, + "label": "CALL", + "name": "get", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "" + }, + { + "id": 154, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "flow-summary" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + }, + { + "id": 203, + "label": "CALL", + "name": "open", + "fullName": "__builtin.open", + "signature": "", + "isExternal": true, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 10, + "tags": "file-io" + } + ], + "purls": [] + }, + { + "flows": [ + { + "id": 155, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "requested_file = tmp0 = request.args\ntmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 5, + "tags": "framework-input" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + }, + { + "id": 203, + "label": "CALL", + "name": "open", + "fullName": "__builtin.open", + "signature": "", + "isExternal": true, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 10, + "tags": "file-io" + } + ], + "purls": [] + }, + { + "flows": [ + { + "id": 147, + "label": "IDENTIFIER", + "name": "tmp0", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0 = request.args", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "framework-input" + }, + { + "id": 153, + "label": "CALL", + "name": "get", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "" + }, + { + "id": 154, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "flow-summary" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + }, + { + "id": 203, + "label": "CALL", + "name": "open", + "fullName": "__builtin.open", + "signature": "", + "isExternal": true, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 10, + "tags": "file-io" + } + ], + "purls": [] + }, + { + "flows": [ + { + "id": 147, + "label": "IDENTIFIER", + "name": "tmp0", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0 = request.args", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "framework-input" + }, + { + "id": 153, + "label": "CALL", + "name": "get", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "" + }, + { + "id": 154, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "flow-summary" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + } + ], + "purls": [] + }, + { + "flows": [ + { + "id": 147, + "label": "IDENTIFIER", + "name": "tmp0", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0 = request.args", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "framework-input" + }, + { + "id": 153, + "label": "CALL", + "name": "get", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "" + }, + { + "id": 154, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 22, + "tags": "flow-summary" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + } + ], + "purls": [] + }, + { + "flows": [ + { + "id": 155, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "requested_file = tmp0 = request.args\ntmp0.get(\"file\")", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 21, + "columnNumber": 5, + "tags": "framework-input" + }, + { + "id": 184, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "not requested_file", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 28, + "columnNumber": 12, + "tags": "" + }, + { + "id": 190, + "label": "IDENTIFIER", + "name": "requested_file", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 35, + "tags": "" + }, + { + "id": 162, + "label": "METHOD_PARAMETER_IN", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "filename", + "typeFullName": "ANY", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 25, + "tags": "flow-summary" + }, + { + "id": 167, + "label": "IDENTIFIER", + "name": "filename", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 49, + "tags": "" + }, + { + "id": 177, + "label": "CALL", + "name": "join", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "" + }, + { + "id": 178, + "label": "BLOCK", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "tmp1.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 16, + "tags": "flow-summary" + }, + { + "id": 179, + "label": "RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "return os.path.join(generator.base_dir, filename)", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 26, + "columnNumber": 9, + "tags": "" + }, + { + "id": 163, + "label": "METHOD_RETURN", + "name": "", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "RET", + "typeFullName": "", + "parentMethodName": "build_file_path", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 24, + "columnNumber": 5, + "tags": "flow-summary" + }, + { + "id": 192, + "label": "CALL", + "name": "build_file_path", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "isExternal": false, + "code": "build_file_path(requested_file)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 31, + "columnNumber": 19, + "tags": "" + }, + { + "id": 201, + "label": "IDENTIFIER", + "name": "target_path", + "fullName": "", + "signature": "", + "isExternal": false, + "code": "open(target_path)", + "typeFullName": "", + "parentMethodName": "handle_request", + "parentMethodSignature": "", + "parentFileName": "app.py", + "parentPackageName": "", + "parentClassName": "", + "lineNumber": 33, + "columnNumber": 15, + "tags": "file-io" + } + ], + "purls": [] + } +] \ No newline at end of file diff --git a/tests/data/atom_reachability/python/atom_reachability.python.usages.slices.json b/tests/data/atom_reachability/python/atom_reachability.python.usages.slices.json new file mode 100644 index 0000000..d6ce7a0 --- /dev/null +++ b/tests/data/atom_reachability/python/atom_reachability.python.usages.slices.json @@ -0,0 +1,1534 @@ +{ + "objectSlices": [ + { + "code": "", + "fullName": "app.py:", + "signature": "", + "fileName": "app.py", + "lineNumber": 1, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "def ", + "isExternal": true, + "lineNumber": 10, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "def ", + "isExternal": true, + "lineNumber": 10, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "app", + "typeFullName": "flask.py:.Flask", + "lineNumber": 5, + "columnNumber": 1, + "label": "LOCAL" + }, + "definedBy": { + "name": "Flask", + "typeFullName": "ANY", + "resolvedMethod": "flask.py:.Flask.__init__", + "isExternal": null, + "lineNumber": 5, + "columnNumber": 7, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [ + { + "callName": "route", + "resolvedMethod": "flask.py:.Flask.route", + "paramTypes": [ + "ANY" + ], + "returnType": "ANY", + "position": 0, + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2 + } + ] + }, + { + "targetObj": { + "name": "route", + "typeFullName": "ANY", + "resolvedMethod": "flask.py:.Flask.route", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "definedBy": { + "name": "route", + "typeFullName": "ANY", + "resolvedMethod": "flask.py:.Flask.route", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": ".assignment", + "typeFullName": "ANY", + "resolvedMethod": ".assignment", + "isExternal": true, + "lineNumber": 37, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": ".assignment", + "typeFullName": "ANY", + "resolvedMethod": ".assignment", + "isExternal": true, + "lineNumber": 37, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.route", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.route", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:", + "isExternal": false, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:", + "isExternal": false, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "Flask", + "typeFullName": "ANY", + "resolvedMethod": "flask.py:.Flask.__init__", + "isExternal": true, + "lineNumber": 5, + "columnNumber": 7, + "label": "CALL" + }, + "definedBy": { + "name": "Flask", + "typeFullName": "ANY", + "resolvedMethod": "flask.py:.Flask.__init__", + "isExternal": true, + "lineNumber": 5, + "columnNumber": 7, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.", + "signature": "", + "fileName": "app.py", + "lineNumber": 10, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 10, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 10, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 18, + "usages": [ + { + "targetObj": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "definedBy": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "invokedCalls": [ + { + "callName": "__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [ + "ANY", + "ANY" + ], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18 + } + ], + "argToCalls": [] + }, + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": ".alloc", + "typeFullName": "ANY", + "resolvedMethod": ".alloc", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": ".alloc", + "typeFullName": "ANY", + "resolvedMethod": ".alloc", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "__newInstance", + "typeFullName": "ANY", + "lineNumber": 13, + "columnNumber": 18, + "label": "LOCAL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": ".alloc", + "isExternal": null, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "__newInstance = ", + "resolvedMethod": null, + "paramTypes": [], + "returnType": "ANY", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18 + }, + { + "callName": "__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [ + "ANY", + "ANY" + ], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18 + } + ], + "argToCalls": [] + }, + { + "targetObj": { + "name": "cls", + "typeFullName": "app.py:.ReportGenerator", + "position": 0, + "lineNumber": 13, + "columnNumber": 18, + "label": "PARAM" + }, + "definedBy": { + "name": "cls", + "typeFullName": "app.py:.ReportGenerator", + "position": 0, + "lineNumber": 13, + "columnNumber": 18, + "label": "PARAM" + }, + "invokedCalls": [], + "argToCalls": [ + { + "callName": "__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [ + "ANY", + "ANY" + ], + "returnType": "ANY", + "position": 0, + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18 + } + ] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 18, + "usages": [ + { + "targetObj": { + "name": "", + "typeFullName": "app.py:.ReportGenerator", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "app.py:.ReportGenerator", + "resolvedMethod": "app.py:.ReportGenerator.", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "definedBy": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "invokedCalls": [ + { + "callName": "", + "resolvedMethod": "class ReportGenerator", + "paramTypes": [ + "ANY" + ], + "returnType": "ANY", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18 + } + ], + "argToCalls": [] + }, + { + "targetObj": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "class ReportGenerator", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": "", + "typeFullName": "ANY", + "resolvedMethod": "class ReportGenerator", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.__init__", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 5, + "usages": [ + { + "targetObj": { + "name": "ReportGenerator", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": false, + "lineNumber": 20, + "columnNumber": 17, + "label": "CALL" + }, + "definedBy": { + "name": "ReportGenerator", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": false, + "lineNumber": 20, + "columnNumber": 17, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": ".fieldAccess", + "typeFullName": "ANY", + "resolvedMethod": ".fieldAccess", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "definedBy": { + "name": ".fieldAccess", + "typeFullName": "ANY", + "resolvedMethod": ".fieldAccess", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.__init__", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 5, + "usages": [ + { + "targetObj": { + "name": "__init__", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 5, + "label": "CALL" + }, + "definedBy": { + "name": "__init__", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 5, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 2, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "definedBy": { + "name": "base_dir", + "typeFullName": "ANY", + "position": 2, + "lineNumber": 13, + "columnNumber": 24, + "label": "PARAM" + }, + "invokedCalls": [ + { + "callName": "app.py:.ReportGenerator.__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [ + "ANY" + ], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 5 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request", + "signature": "", + "fileName": "app.py", + "lineNumber": 18, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "manager_tmp2", + "typeFullName": "ANY", + "lineNumber": 33, + "columnNumber": 10, + "label": "LOCAL" + }, + "definedBy": { + "name": "open", + "typeFullName": "ANY", + "resolvedMethod": "__builtin.open", + "isExternal": null, + "lineNumber": 33, + "columnNumber": 10, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [ + { + "callName": "", + "resolvedMethod": null, + "paramTypes": [], + "returnType": "ANY", + "position": 0, + "isExternal": null, + "lineNumber": 33, + "columnNumber": 10 + }, + { + "callName": "", + "resolvedMethod": null, + "paramTypes": [], + "returnType": "ANY", + "position": 0, + "isExternal": null, + "lineNumber": 33, + "columnNumber": 10 + } + ] + }, + { + "targetObj": { + "name": "target_path", + "typeFullName": "ANY", + "lineNumber": 31, + "columnNumber": 5, + "label": "LOCAL" + }, + "definedBy": { + "name": "build_file_path", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "isExternal": null, + "lineNumber": 31, + "columnNumber": 19, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "open", + "resolvedMethod": "__builtin.open", + "paramTypes": [ + "ANY" + ], + "returnType": "ANY", + "isExternal": true, + "lineNumber": 33, + "columnNumber": 10 + } + ], + "argToCalls": [] + }, + { + "targetObj": { + "name": "app.route", + "typeFullName": "app.route", + "resolvedMethod": "@app.route(\"/report\")", + "isExternal": false, + "lineNumber": 17, + "columnNumber": 2, + "label": "ANNOTATION" + }, + "definedBy": { + "name": "app.route", + "typeFullName": "app.route", + "resolvedMethod": "@app.route(\"/report\")", + "isExternal": false, + "lineNumber": 17, + "columnNumber": 2, + "label": "ANNOTATION" + }, + "invokedCalls": [ + { + "callName": "app.route", + "resolvedMethod": "@app.route(\"/report\")", + "paramTypes": [], + "returnType": "", + "isExternal": false, + "lineNumber": 17, + "columnNumber": 2 + } + ], + "argToCalls": [] + }, + { + "targetObj": { + "name": "tmp0", + "typeFullName": "args", + "lineNumber": 21, + "columnNumber": 22, + "label": "LOCAL" + }, + "definedBy": { + "name": "request.args", + "typeFullName": "ANY", + "resolvedMethod": ".fieldAccess", + "isExternal": null, + "lineNumber": 21, + "columnNumber": 22, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [ + { + "callName": "get", + "resolvedMethod": null, + "paramTypes": [ + "ANY" + ], + "returnType": "ANY", + "position": 0, + "isExternal": null, + "lineNumber": 21, + "columnNumber": 22 + } + ] + }, + { + "targetObj": { + "name": "read", + "typeFullName": "ANY", + "resolvedMethod": "f.read", + "isExternal": true, + "lineNumber": 34, + "columnNumber": 16, + "label": "CALL" + }, + "definedBy": { + "name": "read", + "typeFullName": "ANY", + "resolvedMethod": "f.read", + "isExternal": true, + "lineNumber": 34, + "columnNumber": 16, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": ".logicalNot", + "typeFullName": "ANY", + "resolvedMethod": ".logicalNot", + "isExternal": true, + "lineNumber": 28, + "columnNumber": 8, + "label": "CALL" + }, + "definedBy": { + "name": ".logicalNot", + "typeFullName": "ANY", + "resolvedMethod": ".logicalNot", + "isExternal": true, + "lineNumber": 28, + "columnNumber": 8, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "open", + "typeFullName": "ANY", + "resolvedMethod": "__builtin.open", + "isExternal": true, + "lineNumber": 33, + "columnNumber": 10, + "label": "CALL" + }, + "definedBy": { + "name": "open", + "typeFullName": "ANY", + "resolvedMethod": "__builtin.open", + "isExternal": true, + "lineNumber": 33, + "columnNumber": 10, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request.build_file_path", + "signature": "", + "fileName": "app.py", + "lineNumber": 24, + "columnNumber": 5, + "usages": [ + { + "targetObj": { + "name": "build_file_path", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "isExternal": false, + "lineNumber": 31, + "columnNumber": 19, + "label": "CALL" + }, + "definedBy": { + "name": "build_file_path", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "isExternal": false, + "lineNumber": 31, + "columnNumber": 19, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + }, + { + "targetObj": { + "name": "tmp1", + "typeFullName": "path", + "lineNumber": 26, + "columnNumber": 16, + "label": "LOCAL" + }, + "definedBy": { + "name": "os.path", + "typeFullName": "ANY", + "resolvedMethod": ".fieldAccess", + "isExternal": null, + "lineNumber": 26, + "columnNumber": 16, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [ + { + "callName": "join", + "resolvedMethod": null, + "paramTypes": [ + "ANY", + "ANY" + ], + "returnType": "ANY", + "position": 0, + "isExternal": null, + "lineNumber": 26, + "columnNumber": 16 + } + ] + }, + { + "targetObj": { + "name": "filename", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 24, + "columnNumber": 25, + "label": "PARAM" + }, + "definedBy": { + "name": "filename", + "typeFullName": "ANY", + "position": 1, + "lineNumber": 24, + "columnNumber": 25, + "label": "PARAM" + }, + "invokedCalls": [ + { + "callName": "join", + "resolvedMethod": null, + "paramTypes": [ + "ANY", + "ANY" + ], + "returnType": "ANY", + "isExternal": null, + "lineNumber": 26, + "columnNumber": 16 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.unrelated_top_level_function", + "signature": "", + "fileName": "app.py", + "lineNumber": 37, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "unrelated_top_level_function", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.unrelated_top_level_function", + "isExternal": false, + "lineNumber": 37, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "unrelated_top_level_function", + "typeFullName": "ANY", + "resolvedMethod": "app.py:.unrelated_top_level_function", + "isExternal": false, + "lineNumber": 37, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:", + "signature": "", + "fileName": "app.py", + "lineNumber": 1, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "flask.py:.Flask.__init__", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "flask.py:.Flask.__init__", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "Flask", + "resolvedMethod": "flask.py:.Flask.__init__", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 5, + "columnNumber": 7 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 18, + "usages": [ + { + "targetObj": { + "name": "class ReportGenerator", + "typeFullName": "", + "resolvedMethod": "class ReportGenerator", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": "class ReportGenerator", + "typeFullName": "", + "resolvedMethod": "class ReportGenerator", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "", + "resolvedMethod": "class ReportGenerator", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 18, + "usages": [ + { + "targetObj": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "definedBy": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 18 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.ReportGenerator.__init__", + "signature": "", + "fileName": "app.py", + "lineNumber": 13, + "columnNumber": 5, + "usages": [ + { + "targetObj": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 5, + "label": "CALL" + }, + "definedBy": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 5, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "app.py:.ReportGenerator.__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 13, + "columnNumber": 5 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:", + "signature": "", + "fileName": "app.py", + "lineNumber": 1, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "def ", + "typeFullName": "", + "resolvedMethod": "def ", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "def ", + "typeFullName": "", + "resolvedMethod": "def ", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "", + "resolvedMethod": "def ", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 10, + "columnNumber": 1 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request", + "signature": "", + "fileName": "app.py", + "lineNumber": 18, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "__init__", + "typeFullName": "", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "ReportGenerator", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 20, + "columnNumber": 17 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request", + "signature": "", + "fileName": "app.py", + "lineNumber": 18, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "build_file_path", + "typeFullName": "", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "build_file_path", + "typeFullName": "", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "build_file_path", + "resolvedMethod": "app.py:.handle_request.build_file_path", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 31, + "columnNumber": 19 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request", + "signature": "", + "fileName": "app.py", + "lineNumber": 18, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "read", + "typeFullName": "", + "resolvedMethod": "f.read", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "read", + "typeFullName": "", + "resolvedMethod": "f.read", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "read", + "resolvedMethod": "f.read", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 34, + "columnNumber": 16 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:.handle_request", + "signature": "", + "fileName": "app.py", + "lineNumber": 18, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "open", + "typeFullName": "", + "resolvedMethod": "__builtin.open", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "open", + "typeFullName": "", + "resolvedMethod": "__builtin.open", + "isExternal": true, + "lineNumber": 18, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "open", + "resolvedMethod": "__builtin.open", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 33, + "columnNumber": 10 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:", + "signature": "", + "fileName": "app.py", + "lineNumber": 1, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "route", + "typeFullName": "", + "resolvedMethod": "flask.py:.Flask.route", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "route", + "typeFullName": "", + "resolvedMethod": "flask.py:.Flask.route", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "route", + "resolvedMethod": "flask.py:.Flask.route", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2 + } + ], + "argToCalls": [] + } + ] + }, + { + "code": "", + "fullName": "app.py:", + "signature": "", + "fileName": "app.py", + "lineNumber": 1, + "columnNumber": 1, + "usages": [ + { + "targetObj": { + "name": "route", + "typeFullName": "", + "resolvedMethod": "app.route", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "definedBy": { + "name": "route", + "typeFullName": "", + "resolvedMethod": "app.route", + "isExternal": true, + "lineNumber": 1, + "columnNumber": 1, + "label": "CALL" + }, + "invokedCalls": [ + { + "callName": "", + "resolvedMethod": "app.route", + "paramTypes": [], + "returnType": "", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2 + } + ], + "argToCalls": [] + } + ] + } + ], + "userDefinedTypes": [ + { + "name": "app.py:.ReportGenerator", + "fields": [ + { + "name": "", + "typeFullName": "app.py:.ReportGenerator.", + "lineNumber": null, + "columnNumber": null, + "label": "LOCAL" + }, + { + "name": "__init__", + "typeFullName": "app.py:.ReportGenerator.__init__", + "lineNumber": 13, + "columnNumber": 5, + "label": "LOCAL" + } + ], + "procedures": [ + { + "callName": "", + "resolvedMethod": "app.py:.ReportGenerator.", + "paramTypes": [ + "ANY" + ], + "returnType": "app.py:.ReportGenerator", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18 + }, + { + "callName": "", + "resolvedMethod": "app.py:.ReportGenerator.", + "paramTypes": [ + "app.py:.ReportGenerator", + "ANY" + ], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 18 + }, + { + "callName": "__init__", + "resolvedMethod": "app.py:.ReportGenerator.__init__", + "paramTypes": [ + "app.py:.ReportGenerator", + "ANY", + "ANY" + ], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 13, + "columnNumber": 5 + } + ], + "fileName": "app.py", + "lineNumber": 10, + "columnNumber": 1 + }, + { + "name": "app.py:.ReportGenerator", + "fields": [ + { + "name": "base_dir", + "typeFullName": "ANY", + "lineNumber": 14, + "columnNumber": 9, + "label": "LOCAL" + }, + { + "name": "__init__", + "typeFullName": "app.py:.ReportGenerator.__init__", + "lineNumber": 13, + "columnNumber": 5, + "label": "LOCAL" + } + ], + "procedures": [ + { + "callName": "", + "resolvedMethod": "app.py:.ReportGenerator.", + "paramTypes": [], + "returnType": "ANY", + "isExternal": false, + "lineNumber": 10, + "columnNumber": 1 + } + ], + "fileName": "app.py", + "lineNumber": 10, + "columnNumber": 1 + }, + { + "name": "route", + "fields": [ + { + "name": "\"/report\"", + "typeFullName": "ANY", + "lineNumber": 17, + "columnNumber": 12, + "label": "LOCAL" + } + ], + "procedures": [ + { + "callName": "route", + "resolvedMethod": "flask.py:.Flask.route", + "paramTypes": [ + "ANY", + "ANY", + "ANY" + ], + "returnType": "ANY", + "isExternal": true, + "lineNumber": null, + "columnNumber": null + }, + { + "callName": "app.route", + "resolvedMethod": "app.route(\"/report\")", + "paramTypes": [ + "app", + "/report" + ], + "returnType": "", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2 + }, + { + "callName": "app.route", + "resolvedMethod": "app.route(\"/report\")", + "paramTypes": [ + "app", + "/report" + ], + "returnType": "", + "isExternal": true, + "lineNumber": 17, + "columnNumber": 2 + } + ], + "fileName": "app.py", + "lineNumber": 17, + "columnNumber": 2 + } + ] +} \ No newline at end of file diff --git a/tests/test_atom_reachability_tools.py b/tests/test_atom_reachability_tools.py new file mode 100644 index 0000000..1ca4cd7 --- /dev/null +++ b/tests/test_atom_reachability_tools.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json +import shutil +from pathlib import Path +from unittest import TestCase + +from scanpipe.models import Project +from scanpipe.pipes import collect_and_create_codebase_resources + +from scanpipe.pipes.reachability import ReachabilityStatus + +from reachablecode.pipes.atom import normalize_file_path, normalize_symbol_name, collect_and_match_resources, \ + UsagesMatcher, ReachablesMatcher + +TEST_DATA_PYTHON = ( + Path(__file__).parent.parent / "data" / "atom_reachability" / "python" +) + +USAGES_JSON = "atom_reachability.python.usages.slices.json" +REACHABLES_JSON = "atom_reachability.python.reachables.slices.json" + +COMMIT_HASH = "07ec0de1964b14bf085a1c9a27ece2b61ab6105c" +VCS_URL = "https://github.com/aboutcode-org/test" + + +class UsagesMatcherTests(TestCase): + """Match patch symbols against the real atom ``usages`` slice for ``app.py``.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.slices = json.loads((TEST_DATA_PYTHON / USAGES_JSON).read_text()) + cls.matcher = UsagesMatcher(cls.slices) + cls.names = cls.matcher.by_file.get("app.py") or set() + + def test_matches_expected_app_py_symbols(self): + self.assertIn("app.py", self.matcher.by_file) + + self.assertIn("app.handle_request.build_file_path", self.names) + self.assertIn("app.handle_request", self.names) + self.assertIn("app.ReportGenerator", self.names) + self.assertIn("app.unrelated_top_level_function", self.names) + + self.assertIn( + "build_file_path", + self.matcher.match_symbols({"build_file_path"}, "app.py"), + ) + self.assertIn( + "handle_request.build_file_path", + self.matcher.match_symbols({"handle_request.build_file_path"}, "app.py"), + ) + self.assertIn( + "app.py::handle_request.build_file_path", + self.matcher.match_symbols( + {"app.py::handle_request.build_file_path"}, "app.py" + ), + ) + self.assertIn( + "app.py::ReportGenerator", + self.matcher.match_symbols({"app.py::ReportGenerator"}, "app.py"), + ) + self.assertIn( + "app.py::ReportGenerator.__init__", + self.matcher.match_symbols({"app.py::ReportGenerator.__init__"}, "app.py"), + ) + self.assertIn( + "build_file_path", + self.matcher.match_symbols({"build_file_path"}, "python/app.py"), + ) + self.assertFalse( + self.matcher.match_symbols({"symbol_that_does_not_exist"}, "app.py") + ) + + +class ReachablesMatcherTests(TestCase): + """Match patch symbols against the real atom ``reachables`` slice for ``app.py``.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.slices = json.loads((TEST_DATA_PYTHON / REACHABLES_JSON).read_text()) + cls.matcher = ReachablesMatcher(cls.slices) + + def test_matches_expected_app_py_symbols(self): + self.assertIn("app.py", self.matcher.by_file) + + self.assertIn( + "build_file_path", + self.matcher.match_symbols({"build_file_path"}, "app.py"), + ) + self.assertIn( + "handle_request.build_file_path", + self.matcher.match_symbols({"handle_request.build_file_path"}, "app.py"), + ) + self.assertIn( + "build_file_path", + self.matcher.match_symbols({"build_file_path"}, "python/app.py"), + ) + self.assertFalse( + self.matcher.match_symbols({"symbol_that_does_not_exist"}, "app.py") + ) + + orphan = ReachablesMatcher( + [{"flows": [{"parentMethodName": "x"}], "purls": []}] + ) + self.assertEqual(0, orphan.file_count) + + +class NormalizeTests(TestCase): + """``normalize_file_path`` and ``normalize_symbol_name``.""" + + def test_normalize_file_path(self): + self.assertEqual("app.py", normalize_file_path("app.py")) + self.assertEqual("python/app.py", normalize_file_path("python/app.py")) + self.assertEqual("app.py", normalize_file_path("./app.py")) + self.assertEqual("app.py", normalize_file_path("./././app.py")) + self.assertEqual("pkg/app.py", normalize_file_path("./pkg/app.py")) + self.assertEqual("pkg/app.py", normalize_file_path("/pkg/app.py")) + self.assertEqual("pkg/app.py", normalize_file_path("//pkg/app.py")) + self.assertEqual("pkg/app.py", normalize_file_path(r"pkg\app.py")) + self.assertEqual("pkg/app.py", normalize_file_path(r".\pkg\app.py")) + self.assertEqual("", normalize_file_path("")) + self.assertEqual("", normalize_file_path(None)) + + def test_normalize_symbol_name(self): + self.assertEqual("build_file_path", normalize_symbol_name("build_file_path")) + self.assertEqual( + "serve_report.build_file_path", + normalize_symbol_name("serve_report.build_file_path"), + ) + self.assertEqual( + "app.handle_request.build_file_path", + normalize_symbol_name("app.py:.handle_request.build_file_path"), + ) + self.assertEqual( + "pkg.app.ReportGenerator.__init__", + normalize_symbol_name("pkg/app.py:.ReportGenerator.__init__"), + ) + + self.assertEqual( + "ReportGenerator.__init__", + normalize_symbol_name("app.py::ReportGenerator.__init__"), + ) + + self.assertEqual( + "app.ReportGenerator.__init__", + normalize_symbol_name("app.py:.ReportGenerator.__init__"), + ) + self.assertEqual( + "app.ReportGenerator.__init__", + normalize_symbol_name( + "app.py:.ReportGenerator.__init__" + ), + ) + self.assertEqual( + "app.ReportGenerator.__init__", + normalize_symbol_name( + "app.py:.ReportGenerator.__init__" + ), + ) + self.assertEqual( + "app.ReportGenerator", + normalize_symbol_name("app.py:.ReportGenerator."), + ) + self.assertEqual("def ", normalize_symbol_name("def ")) + + self.assertEqual("", normalize_symbol_name("")) + self.assertEqual("", normalize_symbol_name("")) + self.assertEqual("", normalize_symbol_name("")) + self.assertEqual("", normalize_symbol_name("")) + + self.assertEqual("", normalize_symbol_name(".equals")) + self.assertEqual("", normalize_symbol_name(".equals")) + self.assertEqual("", normalize_symbol_name("")) + self.assertEqual("", normalize_symbol_name(".assignment")) + self.assertEqual("", normalize_symbol_name(".alloc")) + + +class CollectAndMatchResourcesTests(TestCase): + def setUp(self): + self.project = Project.objects.create(name="Analysis") + self.project.codebase_path.mkdir(parents=True, exist_ok=True) + + def _prepare_resources(self): + shutil.copytree(TEST_DATA_PYTHON, self.project.codebase_path / "python") + collect_and_create_codebase_resources(self.project) + for resource in self.project.codebaseresources.all(): + if resource.path.endswith(".py"): + resource.programming_language = "Python" + resource.save() + + def _load_atom_slices(self): + """Load both atom slice fixtures: usages and reachables.""" + usages = json.loads((TEST_DATA_PYTHON / USAGES_JSON).read_text()) + reachables = json.loads((TEST_DATA_PYTHON / REACHABLES_JSON).read_text()) + return usages, reachables + + def _run_atom( + self, + resources, + usages_slices, + reachables_slices, + vulnerable, + fixed, + advisory_uid, + ): + patch_symbols = { + COMMIT_HASH: { + "Python": { + "vulnerable": { + f"app.py::{key}": meta for key, meta in vulnerable.items() + }, + "fixed": {f"app.py::{key}": meta for key, meta in fixed.items()}, + } + } + } + patches = [ + { + "vcs_url": VCS_URL, + "commit_hash": COMMIT_HASH, + "advisory_uids": [advisory_uid], + } + ] + + collect_and_match_resources( + tool_name="atom", + patches=patches, + patch_symbols=patch_symbols, + usages_slices=usages_slices, + reachables_slices=reachables_slices, + candidate_resources=resources, + ) + + for resource in resources: + resource.refresh_from_db() + return resources + + @staticmethod + def _get_reports(resource): + return (resource.extra_data or {}).get("symbols_reachability") or [] + + def test_build_file_path_reachable_through_reachables_slice(self): + """ + ``build_file_path`` is indexed in the real usages slice and marked + reachable by the real reachables slice, so the report is REACHABLE. + """ + self._prepare_resources() + resource = self.project.codebaseresources.get(path="python/app.py") + self.assertEqual("Python", resource.programming_language) + + usages_slices, reachables_slices = self._load_atom_slices() + self.assertTrue(usages_slices.get("objectSlices")) + self.assertTrue(reachables_slices) + + vulnerable = { + "build_file_path": { + "qualified_name": "handle_request.build_file_path", + "text": "return os.path.join(generator.base_dir, filename)", + } + } + fixed = { + "build_file_path": { + "qualified_name": "handle_request.build_file_path", + "text": ( + "return os.path.join(" + "generator.base_dir, os.path.basename(filename))" + ), + } + } + + self._run_atom( + [resource], + usages_slices, + reachables_slices, + vulnerable, + fixed, + advisory_uid="pypi/app/PYSEC-0003", + ) + + results = self._get_reports(resource) + self.assertEqual(1, len(results)) + result = results[0] + + self.assertEqual(["pypi/app/PYSEC-0003"], result["advisory_uids"]) + self.assertEqual(COMMIT_HASH, result["patch"]["commit_hash"]) + self.assertEqual(VCS_URL, result["patch"]["vcs_url"]) + + symbol_key = "app.py::build_file_path" + self.assertEqual([symbol_key], result["vulnerable_symbols"]) + self.assertEqual([symbol_key], result["fixed_symbols"]) + self.assertEqual(ReachabilityStatus.REACHABLE.value, result["is_reachable"]) + + self.assertEqual(1, len(result["tool_details"])) + detail = result["tool_details"][0] + + self.assertEqual(symbol_key, detail["symbol_name"]) + self.assertTrue(detail["in_usage"]) + self.assertTrue(detail["in_reachable"]) + + self.assertTrue(detail["calls"]) + call_names = {call["call_name"] for call in detail["calls"]} + self.assertIn("build_file_path", call_names) + self.assertIn("open", call_names) + self.assertNotIn("is_test", result) diff --git "a/\342\200\216docker.env" "b/\342\200\216docker.env" new file mode 100644 index 0000000..48d19ef --- /dev/null +++ "b/\342\200\216docker.env" @@ -0,0 +1,10 @@ +POSTGRES_DB=scancodeio +POSTGRES_USER=scancodeio +POSTGRES_PASSWORD=scancodeio +POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 + +SCANCODEIO_DB_HOST=db +SCANCODEIO_RQ_REDIS_HOST=redis +SCANCODEIO_ASYNC=True +SCANCODEIO_WORKSPACE_LOCATION=/var/scancodeio/workspace/ +ATOM_EXECUTABLE=/atom-root/bin/atom \ No newline at end of file