diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 8218396b..00000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -omit = .env/*,.venv/* diff --git a/.editorconfig b/.editorconfig index 9ba866b9..efec38e0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,3 @@ insert_final_newline = true [*.{html,md,js,css,sql}] indent_size = 2 - -[*.py] -indent_size = 4 diff --git a/.gcloudignore b/.gcloudignore index 66d48bd5..60b7b33f 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -13,14 +13,9 @@ .git .gitignore -# Python pycache: -__pycache__/ # Ignored by the build system /setup.cfg -.coverage -env/ -.venv/ node_modules/ static/html diff --git a/.github/linters/.flake8 b/.github/linters/.flake8 deleted file mode 100644 index da9c4fd5..00000000 --- a/.github/linters/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -max-line-length = 120 -per-file-ignores = - # The __init__.py file imports the routes and errors file at bottom - /github/workspace/server/__init__.py:E402,F401 - /tmp/lint/server/__init__.py:E402,F401 diff --git a/.github/linters/.htmlhintrc b/.github/linters/.htmlhintrc index 7359e4f3..fece650f 100644 --- a/.github/linters/.htmlhintrc +++ b/.github/linters/.htmlhintrc @@ -5,10 +5,10 @@ "attr-value-not-empty": false, "attr-no-duplication": true, "attr-unsafe-chars": true, - "doctype-first": false, + "doctype-first": true, "doctype-html5": true, - "empyty-tag-self-close": true, - "id-unique": false, + "empty-tag-self-close": true, + "id-unique": true, "id-class-value": true, "head-script-disabled": false, "href-abs-or-rel": false, diff --git a/.github/linters/.htmlhintrc_morechecks b/.github/linters/.htmlhintrc_morechecks index 68d2080e..fece650f 100644 --- a/.github/linters/.htmlhintrc_morechecks +++ b/.github/linters/.htmlhintrc_morechecks @@ -7,7 +7,7 @@ "attr-unsafe-chars": true, "doctype-first": true, "doctype-html5": true, - "empyty-tag-self-close": true, + "empty-tag-self-close": true, "id-unique": true, "id-class-value": true, "head-script-disabled": false, diff --git a/.github/linters/.python-lint b/.github/linters/.python-lint deleted file mode 100644 index 0624349a..00000000 --- a/.github/linters/.python-lint +++ /dev/null @@ -1,480 +0,0 @@ -[MASTER] -errors-only= - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=no - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=flask, - flask_talisman, - google.api_core.exceptions, - google.auth.exceptions, - google.cloud, - jinja2, - jinja2.ext, - markdown, - matplotlib, - matplotlib.pyplot, - mistune, - pandas, - scour, - werkzeug.exceptions, - werkzeug.routing, - werkzeug.http, - pytest - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[BASIC] - -# Naming style matching correct argument names -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= - -# Naming style matching correct attribute names -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= - -# Naming style matching correct class names -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= - -# Naming style matching correct constant names -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct inline iteration names -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= - -# Naming style matching correct method names -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= - -# Naming style matching correct module names -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/.github/linters/.stylelintrc.json b/.github/linters/.stylelintrc.json index 1827eecd..22ac65a8 100644 --- a/.github/linters/.stylelintrc.json +++ b/.github/linters/.stylelintrc.json @@ -1,41 +1,23 @@ { - "extends": "stylelint-config-standard", - "rules": { - "alpha-value-notation": null, - "at-rule-no-unknown": [ - true, - { - "ignoreAtRules": ["prince-pdf"] - } - ], - "color-function-notation": null, - "comment-empty-line-before": null, - "font-family-no-duplicate-names": [ - true, - { - "ignoreFontFamilyNames": ["monospace"] - } - ], - "length-zero-no-unit": null, - "media-feature-range-notation": "prefix", - "no-descending-specificity": null, - "property-no-unknown": [ - true, - { - "ignoreProperties": ["prince-pdf-display-doc-title","prince-pdf-tag-type"] - } - ], - "selector-pseudo-element-no-unknown": [ - true, - { - "ignorePseudoElements": ["footnote-call","footnote-marker"] - } - ], - "value-keyword-case": [ - "lower", - { - "ignoreKeywords": ["Art","A4","A5", "currentColor"] - } - ] - } + "extends": "stylelint-config-standard", + "rules": { + "alpha-value-notation": null, + "color-function-notation": null, + "comment-empty-line-before": null, + "font-family-no-duplicate-names": [ + true, + { + "ignoreFontFamilyNames": ["monospace"] + } + ], + "length-zero-no-unit": null, + "media-feature-range-notation": "prefix", + "no-descending-specificity": null, + "value-keyword-case": [ + "lower", + { + "ignoreKeywords": ["currentColor"] + } + ] + } } diff --git a/.github/workflows/code-static-analysis.yml b/.github/workflows/code-static-analysis.yml index abc1ede0..3241b35b 100644 --- a/.github/workflows/code-static-analysis.yml +++ b/.github/workflows/code-static-analysis.yml @@ -9,7 +9,7 @@ on: branches: [main] paths: - '**.js' - - '**.py' + schedule: # ┌───────────── minute (0 - 59) # │ ┌───────────── hour (0 - 23) @@ -27,30 +27,14 @@ jobs: CodeQL-Build: # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest runs-on: ubuntu-latest - strategy: - matrix: - language: ['javascript', 'python'] steps: - name: Checkout repository uses: actions/checkout@v7 - - name: Set up Python 3.12 - if: ${{ matrix.language == 'python' }} - uses: actions/setup-python@v7 - with: - python-version: '3.12' - - name: Install dependencies - if: ${{ matrix.language == 'python' }} - run: | - pip install -r requirements.txt; - # Set the `CODEQL-PYTHON` environment variable to the Python executable - # that includes the dependencies - echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV - name: Initialize CodeQL uses: github/codeql-action/init@v4.38.0 with: config-file: ./.github/codeql/codeql-config.yml queries: +security-and-quality - setup-python-dependencies: false - languages: '${{ matrix.language }}' + languages: 'javascript' - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4.38.0 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e6cddd21..fb31d381 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,41 +18,27 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'HTTPArchive/httparchive.org' steps: - - name: Checkout branch - uses: actions/checkout@v7 + - name: Checkout branch + uses: actions/checkout@v7 - - name: Setup Node.js for use with actions - uses: actions/setup-node@v7 - with: - node-version: '24' + - name: Setup Node.js for use with actions + uses: actions/setup-node@v7 + with: + node-version: '24' - - name: Set up Python 3.14 - uses: actions/setup-python@v7 - with: - python-version: '3.14' + - name: Install dependencies + run: npm ci || npm install - - name: Install Node dependencies - run: npm install + - name: Build website + run: npm run build - - name: Install Python dependencies - run: pip install -r requirements.txt + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Build static assets - run: npm run build + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - - name: Update timestamps - run: npm run timestamps - - - name: Generate scripts - run: npm run generate - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - - name: Deploy to App Engine - run: echo "Y" | gcloud app deploy --project httparchive + - name: Deploy to App Engine + run: echo "Y" | gcloud app deploy app.yaml --project httparchive diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index dd07fffa..b1f2320c 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -41,12 +41,8 @@ jobs: VALIDATE_JAVASCRIPT_ES: true VALIDATE_JSON: true VALIDATE_MARKDOWN: true - VALIDATE_PYTHON_PYLINT: true - VALIDATE_PYTHON_FLAKE8: true - VALIDATE_PYTHON_BLACK: true VALIDATE_YAML: true - dependabot: name: Dependabot auto-merge runs-on: ubuntu-latest diff --git a/.github/workflows/predeploy.yml b/.github/workflows/predeploy.yml index 213ef230..3d43aa4e 100644 --- a/.github/workflows/predeploy.yml +++ b/.github/workflows/predeploy.yml @@ -25,10 +25,6 @@ jobs: uses: actions/setup-node@v7 with: node-version: '24' - - name: Set up Python 3.14 - uses: actions/setup-python@v7 - with: - python-version: '3.14' - name: Run the website run: ./tools/scripts/run_and_test_website.sh - name: Update timestamps diff --git a/.github/workflows/test-website.yml b/.github/workflows/test-website.yml index b9b0d634..09c5975e 100644 --- a/.github/workflows/test-website.yml +++ b/.github/workflows/test-website.yml @@ -20,33 +20,37 @@ jobs: with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 + - name: Setup Node.js for use with actions uses: actions/setup-node@v7 with: - node-version: '24' - - name: Set up Python 3.14 - uses: actions/setup-python@v7 - with: - python-version: '3.14' + node-version: '26' + - name: Run the website + env: + KEEP_SERVER_RUNNING: true + SOURCE_MAPS: true run: ./tools/scripts/run_and_test_website.sh - - name: Remove node modules to avoid linting errors - run: rm -rf node_modules + - name: Use more complete checks for generated HTML linting run: cp -f .github/linters/.htmlhintrc_morechecks .github/linters/.htmlhintrc + - name: Lint Generated HTML uses: super-linter/super-linter/slim@v8 env: ENABLE_GITHUB_PULL_REQUEST_SUMMARY_COMMENT: false - FILTER_REGEX_INCLUDE: static/html/.* + FILTER_REGEX_INCLUDE: .*static/html/.* + FILTER_REGEX_EXCLUDE: .*node_modules/.* GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_HTML: true USE_FIND_ALGORITHM: true + - name: Set the list of URLs for Lighthouse to check env: RUN_TYPE: ${{ github.event_name }} COMMIT_SHA: ${{ github.sha }} run: ./tools/scripts/set_lighthouse_urls.sh + - name: Audit URLs using Lighthouse uses: treosh/lighthouse-ci-action@12.6.2 id: LHCIAction @@ -55,6 +59,7 @@ jobs: configPath: .github/lighthouse/lighthouse-config-dev.json uploadArtifacts: true # save results as an action artifacts temporaryPublicStorage: true # upload lighthouse report to the temporary storage + - name: Show Lighthouse outputs run: | # All results by URL: diff --git a/.gitignore b/.gitignore index 699a6fbd..2ebb8e9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,10 @@ .idea/ -env/ -.venv/ node_modules/ -static/js -static/html +public/static/js +public/static/html templates/sitemap.xml *.log -*.pyc **/.DS_Store -.coverage -.idx/dev.nix -.coverage.* +.astro/ +/dist +.firebase/ diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..a45fd52c --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/README.md b/README.md index a29cdfad..e544791f 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,60 @@ -# HTTPArchive.org Web Server on App Engine -New and improved version of [HTTP Archive](https://httparchive.org). This replaces the [legacy version](https://legacy.httparchive.org), the source code for which is still available at [HTTPArchive/legacy.httparchive.org](https://github.com/HTTPArchive/legacy.httparchive.org). +# HTTPArchive.org Website ## Run Locally -[Source](https://cloud.google.com/appengine/docs/flexible/python/quickstart) +This is an Astro-based web application for HTTPArchive.org. It requires **Node.js >= 24.0.0**. -1. If you don't have virtualenv, install it using pip. +1. Install the NPM dependencies: -``` -sudo pip install virtualenv -``` + ```bash + npm install + ``` -2. Create an isolated Python environment, and install dependencies: +2. Run the application: -``` -virtualenv --python python3.14 .venv -source .venv/bin/activate -pip install -r requirements.txt -``` + * **Development mode (with live reload):** -3. Install the NPM dependencies: + ```bash + npm run astro:dev + ``` -``` -npm install -``` + In your web browser, open [http://localhost:4321](http://localhost:4321) -4. Initialize the Google Cloud CLI (this is necessary because the App Engine server uses Google Cloud APIs): + * **Production build:** -``` -gcloud init -``` + ```bash + npm run build + ``` -5. Run the application: + * **Build & run locally:** -``` -npm run start -``` + ```bash + npm run start + ``` + +## Linting + +To run the GitHub Super-Linter locally using Docker: -Note: Windows users may need to run `npm run watch` and `python main.py` separately. +* **macOS / Linux:** -6. In your web browser, enter the following address: http://127.0.0.1:8080 + ```bash + npm run lint:darwin:linux + ``` + +* **Windows (Command Prompt):** + + ```bash + npm run lint:win32 + ``` ## Staging (Only available to maintainers) -To test changes on a GCP App Engine server without deploying to the production instance, use the staging app at https://staging-dot-httparchive.uk.r.appspot.com/ +To deploy and test changes on the staging channel: -``` +```bash npm run stage ``` @@ -55,15 +62,15 @@ npm run stage (Only available to maintainers) -To push changes live to the production instance, use the deployment script. Changes will be available on https://httparchive.org. +To push changes live to production instance: -``` +```bash npm run deploy ``` -## Added new icons +## Adding New Icons -We use [Font Awesome](https://fontawesome.com/) icons but inline them directly in the code to save using the JavaScript library. To add, or change an icon, [find one on the Font Awesome website](https://fontawesome.com/icons) and copy the path from the SVG tab and reference it by name in the [reports.json](/config/reports.json) and add the SVG path in the [reports.html](/templates/reports.html) file as per the others. The width in the `reports.json` can be set based on display preferences. +We use [Font Awesome](https://fontawesome.com/) icons but inline them directly in the SVG sprite to avoid using the JavaScript library. To add or change an icon, find one on the [Font Awesome website](https://fontawesome.com/icons), copy the path from the SVG tab, reference it by name in [`config/reports.json`](/config/reports.json), and add the SVG `` path in [`src/pages/reports/index.astro`](/src/pages/reports/index.astro). ## Third-party software licenses diff --git a/app.yaml b/app.yaml index 6238b961..55760020 100644 --- a/app.yaml +++ b/app.yaml @@ -1,48 +1,68 @@ -runtime: python312 -entrypoint: gunicorn -b :$PORT main:app -default_expiration: 3h +runtime: nodejs24 -handlers: +instance_class: F1 + +# Global error handler for 404 pages (serves dist/404.html) +error_handlers: + - file: dist/404.html -# Add longer caching for fonts -- url: /static/fonts/(.*\.(woff|woff2))$ - static_files: static/fonts/\1 - upload: static/fonts/.*\.(woff|woff2)$ - secure: always - expiration: 365d - http_headers: - strict-transport-security: max-age=31556926; includeSubDomains - x-content-type-options: nosniff - vary: Accept-Encoding +handlers: + # 1. Long-term caching for static fonts (/static/fonts/**) + - url: /static/fonts/(.*\.(woff|woff2))$ + static_files: dist/static/fonts/\1 + upload: dist/static/fonts/.*\.(woff|woff2)$ + secure: always + expiration: 365d + http_headers: + Cache-Control: "public, max-age=31536000, immutable" + X-Frame-Options: SAMEORIGIN + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin + Strict-Transport-Security: max-age=31556926; includeSubDomains -# Set the UTF-8 charset for CSS and JS files. -# Makes the assumption that only CSS and JS files existing in the relevant dirs -# but avoids complex regexes -- url: /static/css - static_dir: static/css - secure: always - http_headers: - content-type: text/css; charset=utf-8 - strict-transport-security: max-age=31556926; includeSubDomains - x-content-type-options: nosniff - vary: Accept-Encoding -- url: /static/js - static_dir: static/js - secure: always - http_headers: - content-type: application/javascript; charset=utf-8 - strict-transport-security: max-age=31556926; includeSubDomains - x-content-type-options: nosniff - vary: Accept-Encoding + # 2. Direct static assets pass-through (JS, CSS, images, json, xml, txt, etc.) + - url: /(.*\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|json|xml|txt))$ + static_files: dist/\1 + upload: dist/(.*\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|json|xml|txt))$ + secure: always + http_headers: + X-Frame-Options: SAMEORIGIN + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin + Strict-Transport-Security: max-age=31556926; includeSubDomains -- url: /static - static_dir: static - secure: always - http_headers: - strict-transport-security: max-age=31556926; includeSubDomains - x-content-type-options: nosniff - vary: Accept-Encoding + # 3. Root homepage + - url: / + static_files: dist/index.html + upload: dist/index.html + secure: always + http_headers: + X-Frame-Options: SAMEORIGIN + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin + Strict-Transport-Security: max-age=31556926; includeSubDomains + Content-Security-Policy: >- + default-src 'self' https:; + script-src 'self' https: 'unsafe-inline'; + style-src 'self' https: 'unsafe-inline'; + img-src 'self' https: data:; + font-src 'self' https: data:; + connect-src 'self' https:; -- url: /.* - script: auto - secure: always + # 4. Clean URLs routing (e.g. /about -> dist/about/index.html) + - url: /(.*) + static_files: dist/\1/index.html + upload: dist/(.*)/index.html + secure: always + http_headers: + X-Frame-Options: SAMEORIGIN + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin + Strict-Transport-Security: max-age=31556926; includeSubDomains + Content-Security-Policy: >- + default-src 'self' https:; + script-src 'self' https: 'unsafe-inline'; + style-src 'self' https: 'unsafe-inline'; + img-src 'self' https: data:; + font-src 'self' https: data:; + connect-src 'self' https:; diff --git a/astro.config.mjs b/astro.config.mjs new file mode 100644 index 00000000..6483150f --- /dev/null +++ b/astro.config.mjs @@ -0,0 +1,27 @@ +import { defineConfig } from 'astro/config'; + +const sourceMaps = process.env.SOURCE_MAPS === 'true'; + +export default defineConfig({ + output: 'static', + outDir: 'dist', + + // Base URL + site: 'https://httparchive.org', + devToolbar: { + enabled: false, + }, + server: { + port: 8080, + host: true, + }, + build: { + // Don't add trailing slashes to output filenames + format: 'directory', + }, + vite: { + build: { + sourcemap: sourceMaps, + }, + }, +}); diff --git a/config/last_updated.json b/config/last_updated.json index ed647b8d..173aa441 100644 --- a/config/last_updated.json +++ b/config/last_updated.json @@ -1,223 +1,23 @@ { - "/static/components/summary.css": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "304aa7ab9c39f9eb730f4f9de0261652" - }, - "/static/components/table.css": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2024-04-29T00:00:00.000Z", - "hash": "e2936df46d4db4c6739f21ae2cf3833c" - }, - "/static/css/bootstrap.min.css": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2021-04-23T00:00:00.000Z", - "hash": "ec3bb52a00e176a7181d454dffaea219" - }, - "/static/css/bootstrap.min.css.map": { - "date_published": "2021-09-19T00:00:00.000Z", - "date_modified": "2021-09-19T00:00:00.000Z", - "hash": "35b79ebe0b7805c1c84524ad920faa33" - }, - "/static/css/index.css": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2021-07-20T00:00:00.000Z", - "hash": "7c3287d4a7e4cb6574375131255aff18" - }, - "/static/css/report.css": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2023-09-07T00:00:00.000Z", - "hash": "8b92ae537215eeaecfc9c8a574c836b4" - }, - "/static/css/reports.css": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2023-09-07T00:00:00.000Z", - "hash": "82818febf0ba6c7f255463cdebfa1b12" - }, - "/static/css/styles.css": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2025-06-02T00:00:00.000Z", - "hash": "8f7b318621730dd02645db8f79a27640" - }, - "/static/css/techreport/general.css": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2025-04-16T00:00:00.000Z", - "hash": "fa9ac35ce7d238abc5cfdca8404defa4" - }, - "/static/css/techreport/landing.css": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2025-06-02T00:00:00.000Z", - "hash": "b08e41ab82aee1f65dab08b4c50f1756" - }, - "/static/css/techreport/techreport.css": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2026-04-19T00:00:00.000Z", - "hash": "203ec5059e1b972a22ff05b87780872f" - }, - "/static/js/accessibility.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "78966303745305d33cbf19e9d09b259d" - }, - "/static/js/accessibility.js.map": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "c2cbce2ba7a3898335fde135b24e301d" - }, - "/static/js/export-data.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "8bf4bd3a46a407db8981da5ccc6802e3" - }, - "/static/js/exporting.js": { - "date_published": "2021-05-19T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "2d92935c9a67ad644811dd0cb7c59dd8" - }, - "/static/js/exporting.js.map": { - "date_published": "2024-04-05T00:00:00.000Z", - "date_modified": "2024-04-05T00:00:00.000Z", - "hash": "90a638c281c291abf8acb7531024d3a5" - }, - "/static/js/faq.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "6cc4de9aeed86421f03dcdcaade3f594" - }, - "/static/js/highcharts-more.js": { - "date_published": "2021-04-26T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "6aeff6fa017e791c3a2a81abe0163b12" - }, - "/static/js/highcharts-more.js.map": { - "date_published": "2024-04-05T00:00:00.000Z", - "date_modified": "2024-04-05T00:00:00.000Z", - "hash": "8d31b26d9fd75b9f2749cf0fc652fb34" - }, - "/static/js/highcharts.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "2bc563cd141851943c15d52a34379894" - }, - "/static/js/highstock.js": { - "date_published": "2021-04-26T00:00:00.000Z", - "date_modified": "2024-09-05T00:00:00.000Z", - "hash": "76ffeccbe92601f700b569d7cc51b2da" - }, - "/static/js/highstock.js.map": { - "date_published": "2024-04-05T00:00:00.000Z", - "date_modified": "2024-04-05T00:00:00.000Z", - "hash": "6a6de9a402c29ea5b76e30b5eed6ddd9" - }, - "/static/js/histogram.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "7e254029f57eba242c175b08e02093f2" - }, - "/static/js/index.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "4b870e07e32071f7f286f0559c1a4a32" - }, - "/static/js/main.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "2f99a9c6d75ff31d78aae669fb17dbea" - }, - "/static/js/report.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "8cc05954e531e0174117eb8e4701f5ef" - }, - "/static/js/send-web-vitals.js": { - "date_published": "2022-01-03T00:00:00.000Z", - "date_modified": "2025-06-02T00:00:00.000Z", - "hash": "ab3c52a208f6ed195272017124837a54" - }, - "/static/js/summaryLinked.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "b00f52e619f9aa6b965ebe2a49b6fd23" - }, - "/static/js/tableCWVOverviewMulti.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "ab2994dc074ff02e055c5fb5c8df988b" - }, - "/static/js/tableGeneral.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "d2317a0b43a4c04a86723b178dfedf2c" - }, - "/static/js/tableGeneralMulti.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "62b2c943393389849758cc3d7ff7be5b" - }, - "/static/js/tableOverview.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "0a9bf98ecfeeb9594fde313ae8ab8edb" - }, - "/static/js/tableOverviewMulti.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2023-10-09T00:00:00.000Z", - "hash": "6d90ac2a5081aebcf8377f37dc2d98dc" - }, - "/static/js/techreport.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2026-06-09T00:00:00.000Z", - "hash": "d7124ced4c0de9b8fff69bd6557451a4" - }, - "/static/js/techreport/cwvDistribution.js": { - "date_published": "2026-04-07T00:00:00.000Z", - "date_modified": "2026-04-20T00:00:00.000Z", - "hash": "d9721df034b07cd99460790b04aa711c" - }, - "/static/js/techreport/geoBreakdown.js": { - "date_published": "2026-03-24T00:00:00.000Z", - "date_modified": "2026-04-19T00:00:00.000Z", - "hash": "1cd3ff621d8dcf69965f2c9ed07e0319" - }, - "/static/js/techreport/section.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2026-04-07T00:00:00.000Z", - "hash": "c813fe60fb1bcd338221f72b64739701" - }, - "/static/js/techreport/timeseries.js": { - "date_published": "2023-10-09T00:00:00.000Z", - "date_modified": "2026-04-20T00:00:00.000Z", - "hash": "174a513cecbcd2abfdfe1d4fdbd739e1" - }, - "/static/js/timeseries.js": { - "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "33b1fc3aaf5741c2ec3dd21d56325998" - }, - "/static/js/web-vitals.js": { - "date_published": "2022-01-03T00:00:00.000Z", - "date_modified": "2026-06-09T00:00:00.000Z", - "hash": "2b4eece214d909ad20830bb41ef8ff65" - }, "about.html": { "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2025-06-02T00:00:00.000Z", - "hash": "af69d0e1acd55a05d78630155271b229" + "date_modified": "2026-07-02T00:00:00.000Z", + "hash": "ef6555255ad26e4e47da7186be9bd999" }, "faq.html": { "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2025-08-18T00:00:00.000Z", - "hash": "f14213418e3cd6f4f5b12780f58f3786" + "date_modified": "2026-06-29T00:00:00.000Z", + "hash": "474a30967e56a20920b1aed21c276923" }, "index.html": { "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2026-01-24T00:00:00.000Z", - "hash": "12e34e07b9c530f20cc4b108d07c65a4" + "date_modified": "2026-08-10T00:00:00.000Z", + "hash": "e12fad43d5aaa52dba0f32e7101f0f39" }, "reports.html": { "date_published": "2018-05-08T00:00:00.000Z", - "date_modified": "2023-12-07T00:00:00.000Z", - "hash": "383eb3f292743eb357f80ce20259ee5d" + "date_modified": "2026-07-02T00:00:00.000Z", + "hash": "2b2e2146739bca46032888ec345d5dc2" }, "reports/accessibility": { "date_published": "2018-05-08T00:00:00.000Z", diff --git a/docs/cwv_tech_report.md b/docs/cwv_tech_report.md index 5632703e..095eb12d 100644 --- a/docs/cwv_tech_report.md +++ b/docs/cwv_tech_report.md @@ -1,110 +1,179 @@ -# CWV Tech Report +# CWV Tech Report Architecture & Frontend Guide + +## 1. Architecture Overview + +The HTTP Archive Tech Report is built using **Astro (SSR & client scripting)**, vanilla JavaScript, and **Highcharts**. The initial page skeleton and metadata slots are generated server-side, while data fetching, state synchronization, and visualization rendering happen client-side. + +### Core File Structure + +- **Configuration**: + - `config/techreport.json`: Central configuration containing metric definitions, endpoints, page structures, summary cards, and brackets. +- **Astro Pages**: + - `src/pages/reports/techreport/landing.astro`: Report landing page linking to featured categories, technologies, and comparison entry points. + - `src/pages/reports/techreport/tech.astro`: Polymorphic route matching `/reports/techreport/tech`. Renders skeletons for both **Drilldown** (1 technology) and **Comparison** (2+ technologies) layouts, dynamically pruning the inactive container on load based on `?tech=`. + - `src/pages/reports/techreport/category.astro`: Dedicated category reporting page matching `/reports/techreport/category`. + - `src/pages/reports/techreport/comparison.astro` & `drilldown.astro`: Preserving redirects pointing to `/reports/techreport/tech`. +- **Astro UI Components (`src/components/techreport/`)**: + - `Filters.astro`: Primary sidebar filter form and metadata summary list (`