From a649387213224723736bbd86b660984512855ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20H=C3=A9zs=C5=91?= Date: Sun, 25 Oct 2026 13:48:28 +0100 Subject: [PATCH] drop the pdf closing page and state failed uploads and skipped state resources --- cloudexit/engine.py | 17 +- cloudexit/platform/upload.py | 8 +- cloudexit/report/context.py | 34 +- cloudexit/report/evidence.py | 35 ++ cloudexit/report/executive_summary.py | 3 - cloudexit/report/html_sections.py | 37 +-- cloudexit/report/notices.py | 29 +- cloudexit/report/sections.py | 91 +---- cloudexit/report/wording.py | 89 ++--- tests/cli/test_upload_modes.py | 2 +- tests/fixtures/samples/azure_uploaded.json | 2 +- tests/fixtures/samples/egress_unsized.json | 83 +++++ tests/fixtures/samples/empty_with_costs.json | 27 ++ tests/fixtures/samples/legacy_catalogue.json | 43 +++ .../tfstate/sample_aws_skipped.tfstate | 126 +++++++ tests/fixtures/samples/tfstate_skipped.json | 15 + tests/fixtures/samples/upload_failed.json | 52 +++ tests/report/test_basic_report.py | 314 ++++++++++++++---- tests/report/test_html_report.py | 79 ++++- tests/report/test_report_flow.py | 42 ++- tests/samples/render.py | 19 +- tests/samples/scenarios.py | 24 +- tests/samples/test_samples.py | 90 ++++- tests/samples/test_scenarios.py | 47 +++ tests/test_engine.py | 27 +- 25 files changed, 1061 insertions(+), 274 deletions(-) create mode 100644 tests/fixtures/samples/egress_unsized.json create mode 100644 tests/fixtures/samples/empty_with_costs.json create mode 100644 tests/fixtures/samples/legacy_catalogue.json create mode 100644 tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate create mode 100644 tests/fixtures/samples/tfstate_skipped.json create mode 100644 tests/fixtures/samples/upload_failed.json diff --git a/cloudexit/engine.py b/cloudexit/engine.py index 5507910..15a4390 100644 --- a/cloudexit/engine.py +++ b/cloudexit/engine.py @@ -415,15 +415,19 @@ def perform_risk_assessment(exit_strategy: int, report_path: str) -> dict[str, A return {"success": False, "logs": str(e)} -def _load_tfstate_scope(raw_data_path: str) -> dict[str, Any] | None: - # A missing or unreadable manifest must not fail report generation. +def _load_tfstate_scope( + raw_data_path: str, +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + # A missing or unreadable manifest must not fail report generation. Older + # manifests have no coverage. manifest_path = os.path.join(raw_data_path, "tfstate_manifest.json") try: with open(manifest_path, "r", encoding="utf-8") as manifest_file: - return json.load(manifest_file).get("scope") + manifest = json.load(manifest_file) + return manifest.get("scope"), manifest.get("coverage") except (OSError, json.JSONDecodeError) as e: logger.warning("Could not read the tfstate manifest for report scope: %s", e) - return None + return None, None # Stage 8 @@ -477,10 +481,10 @@ def generate_report( reports = {} # One context feeds all three reports, so they say the same things - tfstate_scope = ( + tfstate_scope, tfstate_coverage = ( _load_tfstate_scope(raw_data_path) if provider_details.get("tfstatePath") - else None + else (None, None) ) provenance = collect_provenance(db_path, run_id=sync_id, upload=upload) context = build_context( @@ -496,6 +500,7 @@ def generate_report( alternative_technology_organizations=alternative_technology_organizations, exit_strategy=exit_strategy, tfstate_scope=tfstate_scope, + tfstate_coverage=tfstate_coverage, provenance=provenance, report_path=report_path, ) diff --git a/cloudexit/platform/upload.py b/cloudexit/platform/upload.py index 33ac4c1..c63ee36 100644 --- a/cloudexit/platform/upload.py +++ b/cloudexit/platform/upload.py @@ -250,7 +250,7 @@ def read_sync_files( return found, problems -REPORT_KEYS = ("status", "reason", "sync_id", "assessment_id", "url") +REPORT_KEYS = ("status", "reason", "sync_id", "assessment_id", "url", "spooled") def _record(result: UploadResult | None) -> dict[str, Any] | None: @@ -273,9 +273,9 @@ def _record(result: UploadResult | None) -> dict[str, Any] | None: return {key: value for key, value in info.items() if value} -def report_info(result: UploadResult | None) -> dict[str, str] | None: - # What the local report may show: the status and the link, nothing the - # platform computed. +def report_info(result: UploadResult | None) -> dict[str, Any] | None: + # What the local report may show: the status, the link and whether a failed + # run was kept, nothing the platform computed. record = _record(result) if record is None: return None diff --git a/cloudexit/report/context.py b/cloudexit/report/context.py index da43ca6..2de9e54 100644 --- a/cloudexit/report/context.py +++ b/cloudexit/report/context.py @@ -38,8 +38,10 @@ class Provenance: uploaded: bool = False platform_url: str | None = None assessment_id: str | None = None - # Why a requested upload did not happen, when it did not + # A requested upload that failed, why, and whether the run was kept + upload_failed: bool = False upload_reason: str | None = None + upload_spooled: bool = False @property def run_short_id(self): @@ -79,9 +81,11 @@ def collect_provenance(db_path, *, run_id=None, upload=None, collected_at=None): uploaded=upload.get("status") == "uploaded", platform_url=upload.get("url") if upload.get("status") == "uploaded" else None, assessment_id=upload.get("assessment_id"), + upload_failed=upload.get("status") == "failed", upload_reason=( - upload.get("reason") if upload.get("status") != "uploaded" else None + upload.get("reason") if upload.get("status") == "failed" else None ), + upload_spooled=bool(upload.get("spooled")) and upload.get("status") == "failed", ) @@ -147,6 +151,14 @@ class ScopeFacts: resource_group: str = "" file_name: str = "" locations: tuple = () + # State file instances left out: of other providers, of types not in the + # data catalogue + not_assessed_other: int = 0 + not_assessed_unmapped: int = 0 + + @property + def not_assessed(self): + return self.not_assessed_other + self.not_assessed_unmapped @dataclass @@ -316,7 +328,14 @@ def _alternative_entries( return entries -def scope_facts(provider_details, cloud_service_provider, tfstate_scope): +def _excluded(coverage, key): + value = (coverage or {}).get(key) + return value if isinstance(value, int) and value > 0 else 0 + + +def scope_facts( + provider_details, cloud_service_provider, tfstate_scope, tfstate_coverage=None +): provider = PROVIDER_NAMES.get(cloud_service_provider, "–") if provider_details.get("tfstatePath"): scope = tfstate_scope or { @@ -343,6 +362,12 @@ def scope_facts(provider_details, cloud_service_provider, tfstate_scope): tuple(rows), file_name=scope.get("file") or "", locations=locations, + not_assessed_other=_excluded( + tfstate_coverage, "instances_excluded_other_provider" + ), + not_assessed_unmapped=_excluded( + tfstate_coverage, "instances_excluded_unmapped" + ), ) if cloud_service_provider == 1: rows = ( @@ -379,6 +404,7 @@ def build_context( alternative_technology_organizations: list[dict[str, Any]], exit_strategy: int, tfstate_scope: dict[str, Any] | None, + tfstate_coverage: dict[str, Any] | None = None, provenance: Provenance, report_path: str, ) -> ReportContext: @@ -392,7 +418,7 @@ def build_context( provider=PROVIDER_NAMES.get(csp, "–"), exit_strategy=exit_strategy, strategy_code=STRATEGY_CODES.get(int(exit_strategy), ""), - scope=scope_facts(provider_details, csp, tfstate_scope), + scope=scope_facts(provider_details, csp, tfstate_scope, tfstate_coverage), provenance=provenance, resources=resources, costs=_cost_entries(cost_data, provenance.collected_at), diff --git a/cloudexit/report/evidence.py b/cloudexit/report/evidence.py index e0ea97e..7c81168 100644 --- a/cloudexit/report/evidence.py +++ b/cloudexit/report/evidence.py @@ -53,6 +53,22 @@ def build_document_table( return key_value_table(rows, context.appearance) +def _not_assessed_text(scope, b): + parts = [ + _counted(b, key, n) + for key, n in ( + ("ev_not_assessed_other", scope.not_assessed_other), + ("ev_not_assessed_unmapped", scope.not_assessed_unmapped), + ) + if n + ] + return ", ".join(parts) + + +def _counted(b, key, n): + return b[f"{key}_one"] if n == 1 else b[f"{key}_many"].format(n=n) + + def scope_rows(context, b): # (label, text, monospaced), the same for the PDF and the HTML report scope = context.scope @@ -91,6 +107,8 @@ def scope_rows(context, b): ): if key in values: rows.append((b[label], values[key] or b["ev_not_determinable"], False)) + if scope.not_assessed: + rows.append((b["ev_not_assessed"], _not_assessed_text(scope, b), False)) source = b["ev_source_tfstate" if scope.kind == "tfstate" else "ev_source_live"] rows.append( ( @@ -117,6 +135,19 @@ def link_markup(url, color): return f'{escape(url)}' +def platform_failure(provenance, b): + # What happened to an upload that failed, as it was true when the report was written + if provenance.upload_reason: + text = b["ev_platform_failed"].format( + reason=provenance.upload_reason.rstrip(". ") + ) + else: + text = b["ev_platform_failed_no_reason"] + if provenance.upload_spooled: + text = f"{text} {b['ev_platform_kept']}" + return text + + def build_provenance_table(context, b): provenance = context.provenance if provenance.catalogue_version and provenance.catalogue_sha256: @@ -137,7 +168,11 @@ def build_provenance_table(context, b): text = escape(b["ev_platform_uploaded"]) if provenance.platform_url: text = f"{text}: {link_markup(provenance.platform_url, context.appearance.heading2_color)}" + elif provenance.assessment_id: + text = f"{text} · {escape(b['fa_uploaded_id'].format(assessment_id=provenance.assessment_id))}" rows.append((b["ev_platform_label"], Paragraph(text, KV_VALUE))) + elif provenance.upload_failed: + rows.append((b["ev_platform_label"], platform_failure(provenance, b))) else: rows.append((b["ev_platform_label"], b["ev_platform_none"])) return key_value_table(rows, context.appearance) diff --git a/cloudexit/report/executive_summary.py b/cloudexit/report/executive_summary.py index eb1b4d5..b36e939 100644 --- a/cloudexit/report/executive_summary.py +++ b/cloudexit/report/executive_summary.py @@ -14,7 +14,6 @@ from .sections import ( alternative_technologies, costs_overview, - full_assessment, glossary, introduction, resource_inventory, @@ -45,8 +44,6 @@ def build_content(context, styles, b): content.append(Spacer(1, 18)) content += glossary(context, styles, b) content.append(chapter_break()) - content += full_assessment(context, styles, b) - content.append(chapter_break()) content += legal_notices(context, styles, b) return content diff --git a/cloudexit/report/html_sections.py b/cloudexit/report/html_sections.py index e41ef37..9cd6146 100644 --- a/cloudexit/report/html_sections.py +++ b/cloudexit/report/html_sections.py @@ -1,6 +1,6 @@ from markupsafe import Markup -from .evidence import scope_rows +from .evidence import platform_failure, scope_rows from .helpers import ( format_change, initials, @@ -11,12 +11,9 @@ strategy_name, ) from .html_egress import egress_page -from .notices import text_parts +from .notices import icon_notice, text_parts from .palette import SEVERITIES from .sections import ( - AWS_ICONS_URL, - AWS_TRADEMARK_URL, - AZURE_ICONS_URL, PRODUCT_SITES, REPOSITORY_URL, ) @@ -406,6 +403,15 @@ def _inventory(context, b): ] if scope.kind == "tfstate": caption_parts.append(b["html_inv_not_managed"]) + if scope.not_assessed: + key = ( + "html_inv_not_assessed_one" + if scope.not_assessed == 1 + else "html_inv_not_assessed_many" + ) + caption_parts.append( + b[key].format(n=scope.not_assessed, about=b["html_nav_about"]) + ) if scope.kind == "azure": empty = b["html_inv_empty_resource_group"].format( @@ -544,10 +550,8 @@ def _count_more(b, n): def _platform_cell(context, b): provenance = context.provenance if not provenance.uploaded: - if provenance.upload_reason: - return [ - b["html_platform_not_uploaded"].format(reason=provenance.upload_reason) - ] + if provenance.upload_failed: + return [platform_failure(provenance, b)] return [b["ev_platform_none"]] parts = [b["ev_platform_uploaded"]] if provenance.platform_url: @@ -667,20 +671,7 @@ def _about(context, egress, b): def _notices(context, egress, b): provenance = context.provenance - icons = None - if context.cloud_service_provider == 2: - icons = text_parts( - b["notices_icons_aws"], - { - "icons_url": (b["html_label_aws_icons"], AWS_ICONS_URL), - "guidelines_url": (b["html_label_aws_guidelines"], AWS_TRADEMARK_URL), - }, - ) - elif context.cloud_service_provider == 1: - icons = text_parts( - b["notices_icons_azure"], - {"icons_url": (b["html_label_azure_icons"], AZURE_ICONS_URL)}, - ) + icons = icon_notice(context.cloud_service_provider, b) if context.scope.kind == "tfstate": figures = b["html_figures_tfstate"] else: diff --git a/cloudexit/report/notices.py b/cloudexit/report/notices.py index 60c96d7..2f81551 100644 --- a/cloudexit/report/notices.py +++ b/cloudexit/report/notices.py @@ -37,14 +37,22 @@ def text_parts(template, links, values=None): return parts -def _icon_notice(provider, b): +def icon_notice(provider, b): + # The names are the links: the HTML shows the name, the PDF the name and, in + # brackets, the address if provider == 2: return text_parts( b["notices_icons_aws"], - {"icons_url": AWS_ICONS_URL, "guidelines_url": AWS_TRADEMARK_URL}, + { + "icons": (b["notices_label_aws_icons"], AWS_ICONS_URL), + "guidelines": (b["notices_label_aws_guidelines"], AWS_TRADEMARK_URL), + }, ) if provider == 1: - return text_parts(b["notices_icons_azure"], {"icons_url": AZURE_ICONS_URL}) + return text_parts( + b["notices_icons_azure"], + {"icons": (b["notices_label_azure_icons"], AZURE_ICONS_URL)}, + ) return None @@ -60,7 +68,7 @@ def notices_html(provider, generated_at, b=BLOCKS, egress=False): *([[b["notices_egress"]]] if egress else []), ] trademarks = [[b["notices_trademark_own"]], [b["notices_trademark_others"]]] - icons = _icon_notice(provider, b) + icons = icon_notice(provider, b) if icons: trademarks.append(icons) return { @@ -85,9 +93,20 @@ def notices_html(provider, generated_at, b=BLOCKS, egress=False): } +def _link_markup(label, url, color): + # A link that is not its own address prints the address too, for print + if label == url: + return link_markup(url, color) + return f"{escape(label)} ({link_markup(url, color)})" + + def parts_markup(parts, color): return "".join( - link_markup(part[1], color) if isinstance(part, tuple) else escape(part) + ( + _link_markup(part[0], part[1], color) + if isinstance(part, tuple) + else escape(part) + ) for part in parts ) diff --git a/cloudexit/report/sections.py b/cloudexit/report/sections.py index c9d5584..5df8c54 100644 --- a/cloudexit/report/sections.py +++ b/cloudexit/report/sections.py @@ -7,7 +7,6 @@ from reportlab.pdfbase.pdfmetrics import stringWidth from reportlab.platypus import ( Flowable, - KeepTogether, ListFlowable, ListItem, Paragraph, @@ -17,7 +16,6 @@ ) from .brief_summary import cost_finding -from .evidence import link_markup from .flow import section from .fonts import FONT, FONT_BOLD, FONT_MONO from .placeholder import LabelWithPlaceholder @@ -56,9 +54,6 @@ "RiskCode", fontName=FONT_MONO ) RISK_MUTED_RIGHT = text_style("RiskMutedRight", 8.5, 11, MUTED, alignment=TA_RIGHT) -COMMAND = text_style("CommandLine", 8.5, 12, INK).clone( - "CommandLine", fontName=FONT_MONO, leftIndent=8, spaceBefore=4, spaceAfter=8 -) GLOSSARY_TERM_WIDTH = 4.2 * cm RISK_BAR_MIN_SEGMENT = 62 ALT_EXAMPLES = 3 @@ -358,6 +353,11 @@ def resource_inventory(context, styles, b): ("NOSPLIT", (0, total_row - 1), (-1, total_row)), ], ) + caption = b["es_res_caption"] + skipped = context.scope.not_assessed + if skipped: + key = "res_not_assessed_one" if skipped == 1 else "res_not_assessed_many" + caption = f"{caption} {b[key].format(n=skipped)}" return section( head, [ @@ -371,7 +371,7 @@ def resource_inventory(context, styles, b): Spacer(1, 12), table, Spacer(1, 6), - Paragraph(b["es_res_caption"], ALT_CAPTION), + Paragraph(caption, ALT_CAPTION), ], ) @@ -576,11 +576,7 @@ def risk_inventory(context, styles, b): def _examples_cell(entry, b): cell = [] for alt in entry.alternatives[:ALT_EXAMPLES]: - vendor = ( - f' · {escape(alt.vendor)}' - if alt.vendor - else "" - ) + vendor = f' · {escape(alt.vendor or b["ev_dash"])}' cell.append(Paragraph(f"{escape(alt.name)}{vendor}", ALT_BODY)) flags = [ b[key] @@ -758,76 +754,3 @@ def glossary(context, styles, b): Spacer(1, 12), table, ] - - -# Closing page -def full_assessment(context, styles, b): - body = styles["ContentStyle"] - provenance = context.provenance - if provenance.uploaded: - content = [ - Paragraph(b["fa_heading_uploaded"], styles["Heading1"]), - Paragraph(escape(b["fa_uploaded"]), body), - ] - if provenance.platform_url: - link = link_markup( - provenance.platform_url, context.appearance.heading2_color - ) - content.append(Paragraph(b["fa_uploaded_link"].format(link=link), body)) - elif provenance.assessment_id: - content.append( - Paragraph( - escape( - b["fa_uploaded_id"].format( - assessment_id=provenance.assessment_id - ) - ), - body, - ) - ) - return content - - rows = [[_text(term, ALT_BODY_BOLD), _text(text)] for term, text in b["fa_items"]] - table = Table( - rows, - colWidths=[GLOSSARY_TERM_WIDTH, 16 * cm - GLOSSARY_TERM_WIDTH], - hAlign="LEFT", - ) - table.setStyle( - TableStyle( - [ - ( - "LINEABOVE", - (0, 0), - (-1, 0), - 1, - HexColor(context.appearance.header_row_color), - ), - ("LINEBELOW", (0, 0), (-1, -1), 0.4, HexColor(HAIRLINE)), - ("LEFTPADDING", (0, 0), (-1, -1), 5), - ("RIGHTPADDING", (0, 0), (-1, -1), 5), - ("TOPPADDING", (0, 0), (-1, -1), 5), - ("BOTTOMPADDING", (0, 0), (-1, -1), 5), - ("VALIGN", (0, 0), (-1, -1), "TOP"), - ] - ) - ) - color = context.appearance.heading2_color - sites = b["fa_sites"].format( - **{name: link_markup(url, color) for name, url in PRODUCT_SITES} - ) - return [ - Paragraph(b["fa_heading"], styles["Heading1"]), - Paragraph(escape(b["fa_lead"]), body), - Spacer(1, 6), - table, - Spacer(1, 6), - KeepTogether( - [ - Paragraph(b["fa_how_heading"], styles["Heading2"]), - Paragraph(escape(b["fa_how"]), body), - Paragraph(escape(b["fa_command"]), COMMAND), - Paragraph(sites, body), - ] - ), - ] diff --git a/cloudexit/report/wording.py b/cloudexit/report/wording.py index 86cc08f..d54befd 100644 --- a/cloudexit/report/wording.py +++ b/cloudexit/report/wording.py @@ -27,6 +27,25 @@ "ev_resource_groups": "Resource Groups", "ev_regions": "Regions", "ev_not_determinable": "Not determinable from the state", + "ev_not_assessed": "Not Assessed", + "ev_not_assessed_other_one": "1 resource of another provider", + "ev_not_assessed_other_many": "{n} resources of other providers", + "ev_not_assessed_unmapped_one": "1 resource of a type not in the data catalogue", + "ev_not_assessed_unmapped_many": ( + "{n} resources of types not in the data catalogue" + ), + "res_not_assessed_one": ( + "1 resource in the state file was not assessed (see page 2)." + ), + "res_not_assessed_many": ( + "{n} resources in the state file were not assessed (see page 2)." + ), + "html_inv_not_assessed_one": ( + "1 resource in the state file was not assessed (see {about})" + ), + "html_inv_not_assessed_many": ( + "{n} resources in the state file were not assessed (see {about})" + ), "ev_source_label": "Source", "ev_source_live": "Provider APIs", "ev_source_tfstate": "Terraform State", @@ -38,6 +57,9 @@ "ev_platform_label": "Platform", "ev_platform_none": "Self-collected data: no platform evaluated this report.", "ev_platform_uploaded": "Uploaded to the platform", + "ev_platform_failed": "Upload failed: {reason}.", + "ev_platform_failed_no_reason": "Upload failed.", + "ev_platform_kept": "The run was kept for a later upload.", "ev_stored_footnote": ( "This document was generated when run {run} completed and is stored " "unchanged. Later runs are reported in their own documents." @@ -335,41 +357,6 @@ "definition": "One completed assessment of the defined scope, identified by the run ID on page 2.", }, ], - "fa_heading": "What the Full Assessment Adds", - "fa_lead": ( - "This Executive Summary is the Basic assessment. It was collected and " - "evaluated locally and covers the resource inventory, " - "costs, exit risks and alternative technologies of the defined scope. " - "The full assessment on a platform adds:" - ), - "fa_items": [ - ( - "Exit Readiness Grade", - "a letter grade from A to F that shows how ready the defined scope is for an exit.", - ), - ( - "Vendor Lock-in Score", - "a rating of how easily each resource type can be replaced, in terms of skills, technology and operations.", - ), - ( - "Scored Alternatives", - "the alternative technologies for each resource type, rated and ranked.", - ), - ], - "fa_how_heading": "How to Get Them", - "fa_how": ( - "Connect cloudexit to a platform with an API key and run the " - "assessment again; the results are uploaded and evaluated there. For " - "example:" - ), - "fa_command": "cloudexit aws --host HOST --api-key-file KEY_FILE", - "fa_sites": "Platforms: {escapecloud} · {exitcloud}", - "fa_heading_uploaded": "The Full Assessment", - "fa_uploaded": ( - "This assessment was uploaded. The Exit Readiness Grade, the Vendor " - "Lock-in Score, scored alternatives and the Egress & Data Landscape " - "are available in its reports on the platform." - ), "fa_uploaded_link": "Open the reports: {link}", "fa_uploaded_id": "Assessment ID: {assessment_id}", "eg_heading_introduction": "Introduction", @@ -652,20 +639,20 @@ "affiliation." ), "notices_icons_aws": ( - "AWS service icons are taken from the AWS Architecture Icons " - "({icons_url}); their use is subject to the terms published there and " - "in the AWS Trademark Guidelines ({guidelines_url}). Amazon Web " - "Services and AWS are trademarks of Amazon.com, Inc. or its " - "affiliates. This report is not endorsed by or affiliated with Amazon " - "Web Services." + "AWS service icons are taken from the {icons}; their use is subject to " + "the terms published there and in the {guidelines}. Amazon Web Services " + "and AWS are trademarks of Amazon.com, Inc. or its affiliates. This " + "report is not endorsed by or affiliated with Amazon Web Services." ), "notices_icons_azure": ( - "Azure service icons are taken from the Azure Architecture Icons " - "({icons_url}); their use is subject to Microsoft's icon terms " - "published there. Microsoft and Azure are trademarks of the Microsoft " - "group of companies. This report is not endorsed by or affiliated with " - "Microsoft." - ), + "Azure service icons are taken from the {icons}; their use is subject " + "to Microsoft's icon terms published there. Microsoft and Azure are " + "trademarks of the Microsoft group of companies. This report is not " + "endorsed by or affiliated with Microsoft." + ), + "notices_label_aws_icons": "AWS Architecture Icons", + "notices_label_aws_guidelines": "AWS Trademark Guidelines", + "notices_label_azure_icons": "Azure Architecture Icons", "notices_software_heading": "Software", "notices_software": ( "cloudexit is free software, licensed under the GNU Affero General " @@ -752,7 +739,6 @@ "and operations, with the alternatives rated and ranked." ), "html_scores_on_platform": "For this run it is on the platform.", - "html_platform_not_uploaded": "Not uploaded: {reason}", "html_hint_title": "EscapeCloud Light & Enterprise", "html_hint_text": ( "EscapeCloud's scoring engine offers several scoring methods to assess " @@ -934,9 +920,6 @@ "html_notice_advice": ( "It is not legal, financial or investment advice, an audit or an exit plan." ), - "html_label_aws_icons": "AWS Architecture Icons", - "html_label_aws_guidelines": "AWS Trademark Guidelines", - "html_label_azure_icons": "Azure Architecture Icons", "html_label_repository": "github.com/escapecloud/cloudexit", "html_figures_heading": "Figures", "html_figures": ( @@ -1049,6 +1032,7 @@ "ev_catalogue_label", "ev_run_id", "ev_platform_label", + "ev_not_assessed", "cover_title_executive_summary", "cover_title_egress", "ev_egress_report", @@ -1143,9 +1127,6 @@ "es_heading_severity", "es_severity_th_meaning", "es_heading_glossary", - "fa_heading", - "fa_how_heading", - "fa_heading_uploaded", "html_download", "html_download_summary", "html_download_egress", diff --git a/tests/cli/test_upload_modes.py b/tests/cli/test_upload_modes.py index f0b3299..7cd86f2 100644 --- a/tests/cli/test_upload_modes.py +++ b/tests/cli/test_upload_modes.py @@ -91,7 +91,7 @@ def test_auto_with_a_platform_that_refuses_the_key_writes_the_report_first(self) self.assertEqual(ctx.exception.code, codes.UPLOAD_AUTH) self.assert_local_report(report) - self.assertIn("Not uploaded", self.html(report)) + self.assertIn("Upload failed", self.html(report)) def test_required_without_a_platform_fails_before_collecting_anything(self): report = self.report() diff --git a/tests/fixtures/samples/azure_uploaded.json b/tests/fixtures/samples/azure_uploaded.json index 6d65415..1131ce0 100644 --- a/tests/fixtures/samples/azure_uploaded.json +++ b/tests/fixtures/samples/azure_uploaded.json @@ -1,6 +1,6 @@ { "name": "Sample – Document Archive", - "purpose": "The same run, uploaded with a link: the uploaded closing page and the link on page 2 of both documents.", + "purpose": "Uploaded with a link: the link on page 2 of both PDFs and in the HTML.", "provider": "azure", "exit_strategy": 1, "scope": { diff --git a/tests/fixtures/samples/egress_unsized.json b/tests/fixtures/samples/egress_unsized.json new file mode 100644 index 0000000..7c3c17e --- /dev/null +++ b/tests/fixtures/samples/egress_unsized.json @@ -0,0 +1,83 @@ +{ + "name": "Sample – Archive Migration Staging", + "purpose": "Egress where no data store has a size: fee and totals without a figure, every type not sized, no free-tier headroom.", + "provider": "azure", + "exit_strategy": 1, + "scope": { + "tenantId": "00000000-0000-4000-8000-0000000000a1", + "subscriptionId": "00000000-0000-4000-8000-0000000000a2", + "resourceGroupName": "rg-sample-staging" + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "region": "westeurope", + "count": 3 + }, + { + "type": "Microsoft.Sql/servers/databases", + "region": "westeurope", + "count": 2 + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "region": "westeurope", + "count": 1 + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "region": "westeurope", + "count": 1 + } + ], + "costs": { + "currency": "EUR", + "months": { + "2026-06": 830.4, + "2026-07": 842.15, + "2026-08": 851.0, + "2026-09": 860.3, + "2026-10": 855.75, + "2026-11": 351.2 + } + }, + "egress": { + "archive_tiers": [ + "Archive" + ], + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "samplestagingblobs", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "samplestagingexports", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-staging-orders", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "sample-staging-search", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "name": "sample-staging-vault", + "size_gib": null + } + ] + }, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0014" +} diff --git a/tests/fixtures/samples/empty_with_costs.json b/tests/fixtures/samples/empty_with_costs.json new file mode 100644 index 0000000..cadea8b --- /dev/null +++ b/tests/fixtures/samples/empty_with_costs.json @@ -0,0 +1,27 @@ +{ + "name": "Sample – Decommissioned Test Environment", + "purpose": "An empty resource group that still shows six months of small costs: empty states next to real figures.", + "provider": "azure", + "exit_strategy": 1, + "scope": { + "tenantId": "00000000-0000-4000-8000-0000000000a1", + "subscriptionId": "00000000-0000-4000-8000-0000000000a2", + "resourceGroupName": "rg-sample-decommissioned" + }, + "resources": [], + "costs": { + "currency": "EUR", + "months": { + "2026-06": 38.2, + "2026-07": 31.75, + "2026-08": 24.4, + "2026-09": 12.9, + "2026-10": 6.35, + "2026-11": 1.8 + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0015" +} diff --git a/tests/fixtures/samples/legacy_catalogue.json b/tests/fixtures/samples/legacy_catalogue.json new file mode 100644 index 0000000..2254651 --- /dev/null +++ b/tests/fixtures/samples/legacy_catalogue.json @@ -0,0 +1,43 @@ +{ + "name": "Sample – Internal Wiki", + "purpose": "A dataset without catalogue_meta: the catalogue version is unknown on page 2 and in About This Report.", + "provider": "aws", + "exit_strategy": 2, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 1 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-central-1", + "count": 1 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 96.4, + "2026-07": 98.1, + "2026-08": 97.25, + "2026-09": 99.6, + "2026-10": 98.8, + "2026-11": 40.3 + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0016", + "catalogue": "legacy" +} diff --git a/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate b/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate new file mode 100644 index 0000000..2724066 --- /dev/null +++ b/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate @@ -0,0 +1,126 @@ +{ + "version": 4, + "terraform_version": "1.14.6", + "serial": 31, + "lineage": "00000000-0000-4000-8000-0000000000c3", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "aws_instance", + "name": "app", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "availability_zone": "eu-central-1a", + "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef0" + } + }, + { + "schema_version": 0, + "attributes": { + "availability_zone": "eu-central-1b", + "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef1" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "assets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "region": "eu-central-1", + "bucket": "sample-landing-assets", + "arn": "arn:aws:s3:::sample-landing-assets" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "deploy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::123456789012:role/sample-deploy" + } + }, + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::123456789012:role/sample-readonly" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "app", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:eu-central-1:123456789012:log-group:/sample/app" + } + } + ] + }, + { + "mode": "managed", + "type": "azurerm_resource_group", + "name": "shared", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "location": "westeurope", + "name": "rg-sample-shared" + } + } + ] + }, + { + "mode": "managed", + "type": "azurerm_storage_account", + "name": "backup", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "location": "westeurope", + "name": "samplebackupstore" + } + } + ] + }, + { + "mode": "managed", + "type": "google_storage_bucket", + "name": "exports", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "location": "EU", + "name": "sample-exports" + } + } + ] + } + ] +} diff --git a/tests/fixtures/samples/tfstate_skipped.json b/tests/fixtures/samples/tfstate_skipped.json new file mode 100644 index 0000000..c156841 --- /dev/null +++ b/tests/fixtures/samples/tfstate_skipped.json @@ -0,0 +1,15 @@ +{ + "name": "Sample – Multi-Cloud Landing Zone (Terraform)", + "purpose": "A state file with instances of another provider and of types not in the data catalogue: the Not Assessed row on page 2 and in About This Report.", + "provider": "aws", + "exit_strategy": 1, + "scope": { + "tfstate": "sample_aws_skipped.tfstate" + }, + "resources": [], + "costs": null, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0017" +} diff --git a/tests/fixtures/samples/upload_failed.json b/tests/fixtures/samples/upload_failed.json new file mode 100644 index 0000000..5ebd294 --- /dev/null +++ b/tests/fixtures/samples/upload_failed.json @@ -0,0 +1,52 @@ +{ + "name": "Sample – Order Tracking Service", + "purpose": "A failed upload that was kept for a later upload: the wording on page 2 and in About This Report.", + "provider": "aws", + "exit_strategy": 1, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 5 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-central-1", + "count": 1 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-central-1", + "count": 6 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 412.5, + "2026-07": 398.2, + "2026-08": 405.75, + "2026-09": 421.1, + "2026-10": 418.9, + "2026-11": 171.6 + } + }, + "egress": null, + "upload": { + "status": "failed", + "reason": "The platform did not answer in time", + "spooled": true, + "sync_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0013" + }, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0013" +} diff --git a/tests/report/test_basic_report.py b/tests/report/test_basic_report.py index 24606d8..64d3e4d 100644 --- a/tests/report/test_basic_report.py +++ b/tests/report/test_basic_report.py @@ -69,15 +69,13 @@ (0, "Risk Severity Levels"), (0, "Glossary"), ] -OFFLINE_OUTLINE = BASE_OUTLINE + [ - (0, "What the Full Assessment Adds"), - (1, "How to Get Them"), - (0, "Legal Notices"), -] -UPLOADED_OUTLINE = BASE_OUTLINE + [ - (0, "The Full Assessment"), - (0, "Legal Notices"), -] +OUTLINE = BASE_OUTLINE + [(0, "Legal Notices")] +CLOSING_PAGE_TEXTS = ( + "What the Full Assessment Adds", + "The Full Assessment", + "How to Get Them", + "HOST", +) SCORE_PATTERNS = ( r"/ 100", r"/ 5\b", @@ -87,6 +85,14 @@ "Lock-in Score", "Scored Alternatives", ) +# An edition name; the "Enterprise support" flag of an alternative is not one +PROMOTION_PATTERNS = ( + r"\bLight\b", + r"\bEnterprise\b(?!\s+support)", + "Exit Readiness Grade", + "Vendor Lock-in Score", +) +EXITCLOUD_URL = "https://app.exitcloud.io/assessments/a-1" TFSTATE_SCOPE = { "file": "infra.tfstate", "sha256": "ab" * 32, @@ -252,19 +258,125 @@ def test_a_tfstate_assessment_explains_the_missing_costs(self): self.assertIn("e1c2f0c0-0000-4000-8000-000000000000 (serial 7)", text) +class TfstateCoverageTests(BasicReportTestCase): + def coverage(self, other=0, unmapped=0): + return { + "instances_total": 9 + other + unmapped, + "instances_counted": 9, + "instances_excluded_other_provider": other, + "instances_excluded_unmapped": unmapped, + } + + def pages(self, key, coverage, **overrides): + _path, context, reader = self.render( + self.tfstate_fixture(), + key=key, + tfstate_scope=TFSTATE_SCOPE, + tfstate_coverage=coverage, + **overrides, + ) + return context, page_texts(reader) + + def test_both_kinds_are_named_with_the_total_in_the_inventory_caption(self): + context, pages = self.pages("covered", self.coverage(other=3, unmapped=2)) + + self.assertEqual(context.scope.not_assessed, 5) + self.assertIn( + "Not Assessed 3 resources of other providers, " + "2 resources of types not in the data catalogue", + pages[1], + ) + inventory = next(text for text in pages if "Share = resources" in text) + self.assertIn( + "5 resources in the state file were not assessed (see page 2)", inventory + ) + + def test_only_the_parts_that_are_not_zero_are_shown(self): + for key, coverage, expected, absent in ( + ( + "other", + self.coverage(other=4), + "4 resources of other providers", + "types not in", + ), + ( + "unmapped", + self.coverage(unmapped=1), + "1 resource of a type not in the data catalogue", + "other provider", + ), + ( + "one other", + self.coverage(other=1), + "1 resource of another provider", + "types not in", + ), + ): + with self.subTest(key): + _context, pages = self.pages(key, coverage) + row = pages[1].split("Not Assessed", 1)[1].split("Source", 1)[0] + + self.assertIn(expected, row) + self.assertNotIn(absent, row) + + def test_a_singular_total_says_was(self): + _context, pages = self.pages("one total", self.coverage(unmapped=1)) + + inventory = next(text for text in pages if "Share = resources" in text) + self.assertIn( + "1 resource in the state file was not assessed (see page 2)", inventory + ) + + def test_without_excluded_instances_there_is_no_row_and_no_caption(self): + for key, coverage in ( + ("zeros", self.coverage()), + ("older manifest", None), + ("empty", {}), + ( + "not numbers", + { + "instances_excluded_other_provider": "3", + "instances_excluded_unmapped": -2, + }, + ), + ): + with self.subTest(key): + context, pages = self.pages(key, coverage) + text = " ".join(pages) + + self.assertEqual(context.scope.not_assessed, 0) + self.assertNotIn("Not Assessed", text) + self.assertNotIn("were not assessed", text) + self.assertNotIn("was not assessed", text) + + def test_a_live_run_never_shows_the_row(self): + _path, context, reader = self.render( + key="live coverage", tfstate_coverage=self.coverage(other=3, unmapped=2) + ) + + self.assertEqual(context.scope.not_assessed, 0) + self.assertNotIn("Not Assessed", " ".join(page_texts(reader))) + + class StructureTests(BasicReportTestCase): def test_the_heading_outline_is_the_expected_list(self): for name, fixture, overrides in self.scenarios(): with self.subTest(name): _path, _context, reader = self.render(fixture, key=name, **overrides) - expected = UPLOADED_OUTLINE if "uploaded" in name else OFFLINE_OUTLINE - self.assertEqual(outline_entries(reader.outline), expected) + self.assertEqual(outline_entries(reader.outline), OUTLINE) + + def test_the_outline_is_the_same_for_an_upload_without_a_link(self): + _path, _context, reader = self.render( + key="no link", provenance=provenance(uploaded=True, assessment_id="a-77") + ) + + self.assertEqual(outline_entries(reader.outline), OUTLINE) def test_the_outline_does_not_depend_on_the_data(self): _path, _context, reader = self.render( build_empty_report_fixture(), provenance=provenance() ) - self.assertEqual(outline_entries(reader.outline), OFFLINE_OUTLINE) + self.assertEqual(outline_entries(reader.outline), OUTLINE) def test_each_chapter_starts_a_page_and_page_two_is_the_document_information(self): _path, _context, reader = self.render(key="default") @@ -281,7 +393,9 @@ def test_each_chapter_starts_a_page_and_page_two_is_the_document_information(sel numbers = [reader.get_destination_page_number(entry) for entry in chapters] # One chapter per page, except the severity levels and the glossary self.assertEqual(len(set(numbers)), len(numbers) - 1) - self.assertEqual(numbers[-4], numbers[-3]) + self.assertEqual(numbers[-3], numbers[-2]) + self.assertIn("Legal Notices", pages[numbers[-1]]) + self.assertEqual(numbers[-1], len(pages) - 1) def test_running_header_and_footer(self): path, context, reader = self.render(key="default") @@ -393,7 +507,7 @@ def test_names_from_the_data_are_escaped(self): class AlternativesTests(BasicReportTestCase): - def test_a_missing_vendor_is_left_out_and_alternatives_are_alphabetical(self): + def test_a_missing_vendor_shows_a_dash_and_alternatives_are_alphabetical(self): fixture = build_report_fixture() technologies = fixture["alternative_technologies"] technologies.append( @@ -416,7 +530,11 @@ def test_a_missing_vendor_is_left_out_and_alternatives_are_alphabetical(self): ) self.assertEqual(entry.alternatives[0].vendor, "") self.assertEqual(entry.alternatives[1].vendor, "OpenInfra Foundation") - self.assertNotIn("Unknown Organization", " ".join(page_texts(reader))) + text = " ".join(page_texts(reader)) + self.assertIn("Alpha · –", text) + self.assertIn("OpenStack · OpenInfra Foundation", text) + self.assertLess(text.index("Alpha · –"), text.index("OpenStack ·")) + self.assertNotIn("Unknown Organization", text) class ProvenanceTests(BasicReportTestCase): @@ -433,33 +551,71 @@ def test_an_offline_report_says_it_is_self_collected(self): self.assertIn(SYNC_ID, second) self.assertIn("2026-11-02 · (sha256 cccccccccccc)", second) self.assertIn("Self-collected data: no platform evaluated this report.", second) - self.assertIn("What the Full Assessment Adds", pages[-2]) self.assertEqual( link_targets(reader) - AWS_NOTICE_LINKS, - {"https://escapecloud.io", "https://exitcloud.io"}, + set(), ) - def test_an_uploaded_report_carries_the_link_instead_of_the_promotion(self): + def test_an_uploaded_report_carries_the_link_on_page_two_only(self): _path, _context, reader = self.render(key="uploaded", provenance=uploaded()) pages = page_texts(reader) self.assertIn("Uploaded to the platform", pages[1]) self.assertIn(URL, pages[1]) self.assertNotIn("Self-collected data", " ".join(pages)) - self.assertIn("The Full Assessment", pages[-2]) - self.assertIn(URL, pages[-2]) - self.assertNotIn("What the Full Assessment Adds", " ".join(pages)) + self.assertEqual(sum(URL in text for text in pages), 1) self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, {URL}) - def test_an_upload_without_a_link_names_the_assessment(self): + def test_an_upload_without_a_link_names_the_assessment_on_page_two(self): _path, _context, reader = self.render( - provenance=( - uploaded(platform_url=None) - if False - else provenance(uploaded=True, platform_url=None, assessment_id="a-77") - ) + key="no link", provenance=provenance(uploaded=True, assessment_id="a-77") + ) + pages = page_texts(reader) + + self.assertIn("Uploaded to the platform", pages[1]) + self.assertIn("Assessment ID: a-77", pages[1]) + self.assertEqual(sum("a-77" in text for text in pages), 1) + self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, set()) + + def test_a_failed_upload_says_what_happened(self): + for name, changes, expected in ( + ( + "kept", + {"upload_reason": "upload_unavailable", "upload_spooled": True}, + "Upload failed: upload_unavailable. The run was kept for a later upload.", + ), + ( + "lost", + {"upload_reason": "upload_rejected"}, + "Upload failed: upload_rejected.", + ), + ("no reason", {}, "Upload failed."), + ( + "no reason, kept", + {"upload_spooled": True}, + "Upload failed. The run was kept", + ), + ): + with self.subTest(name): + _path, _context, reader = self.render( + key=f"failed {name}", + provenance=provenance(upload_failed=True, **changes), + ) + pages = page_texts(reader) + + self.assertIn(expected, pages[1]) + self.assertNotIn("Self-collected data", " ".join(pages)) + self.assertEqual("kept for a later upload" in pages[1], "kept" in name) + self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, set()) + + def test_a_reason_that_ends_in_a_full_stop_is_not_doubled(self): + _path, _context, reader = self.render( + key="failed stop", + provenance=provenance(upload_failed=True, upload_reason="No answer."), ) - self.assertIn("Assessment ID: a-77", page_texts(reader)[-2]) + + self.assertIn("Upload failed: No answer. ", page_texts(reader)[1] + " ") + self.assertNotIn("No answer..", page_texts(reader)[1]) def test_an_unknown_catalogue_version_is_said_so(self): _path, _context, reader = self.render( @@ -554,31 +710,71 @@ def test_the_json_report_has_a_provenance_object(self): class BasicOnlyTests(BasicReportTestCase): - def test_no_score_or_grade_appears_outside_the_closing_page(self): + def test_no_score_or_grade_appears_on_any_page(self): for name, fixture, overrides in self.scenarios(): with self.subTest(name): _path, _context, reader = self.render(fixture, key=name, **overrides) - body = " ".join(page_texts(reader)[:-2]) + body = " ".join(page_texts(reader)) for pattern in SCORE_PATTERNS: self.assertIsNone(re.search(pattern, body, re.IGNORECASE), pattern) - def test_the_closing_page_only_names_what_the_full_assessment_adds(self): - _path, _context, reader = self.render(key="offline", provenance=provenance()) - closing = page_texts(reader)[-2] + def test_no_closing_page_and_no_command_is_left(self): + for name, fixture, overrides in self.scenarios(): + with self.subTest(name): + _path, _context, reader = self.render(fixture, key=name, **overrides) + body = " ".join(page_texts(reader)) - for name in ( - "Exit Readiness Grade", - "Vendor Lock-in Score", - "Scored Alternatives", - ): - self.assertIn(name, closing) - # The Community Edition produces it itself, so the platforms do not add it - self.assertNotIn("Egress & Data Landscape", closing) - self.assertFalse( - any("Egress" in term for term, _text in BLOCKS["fa_items"]), - ) - self.assertNotRegex(closing, r"\d+ / (100|5)") + for text in CLOSING_PAGE_TEXTS: + self.assertNotIn(text, body) + self.assertNotIn("--api-key-file", body) + + def test_no_edition_or_grade_is_promoted_on_any_page(self): + for name, fixture, overrides in self.scenarios(): + with self.subTest(name): + _path, _context, reader = self.render(fixture, key=name, **overrides) + for number, text in enumerate(page_texts(reader), start=1): + for pattern in PROMOTION_PATTERNS: + self.assertIsNone(re.search(pattern, text), (number, pattern)) + + def test_the_enterprise_support_flag_does_not_trip_the_edition_check(self): + _path, _context, reader = self.render(key="default") + body = " ".join(page_texts(reader)) + + self.assertIn("Enterprise support", body) + self.assertIsNone(re.search(PROMOTION_PATTERNS[1], body)) + self.assertIsNotNone(re.search(PROMOTION_PATTERNS[1], "The Enterprise edition")) + self.assertIsNotNone(re.search(PROMOTION_PATTERNS[0], "A Light assessment")) + + def test_exitcloud_is_named_only_on_page_two_of_an_uploaded_report(self): + for name, fixture, overrides in self.scenarios(): + if "uploaded" in name: + overrides = { + **overrides, + "provenance": provenance( + uploaded=True, platform_url=EXITCLOUD_URL, assessment_id="a-1" + ), + } + with self.subTest(name): + _path, _context, reader = self.render(fixture, **overrides) + pages = [text.lower() for text in page_texts(reader)] + named = [ + n for n, text in enumerate(pages, start=1) if "exitcloud" in text + ] + + self.assertEqual(named, [2] if "uploaded" in name else []) + + def test_escapecloud_is_named_only_on_the_legal_notices_page(self): + for name, fixture, overrides in self.scenarios(): + with self.subTest(name): + _path, _context, reader = self.render(fixture, key=name, **overrides) + pages = [text.lower() for text in page_texts(reader)] + named = [ + n for n, text in enumerate(pages, start=1) if "escapecloud" in text + ] + + self.assertEqual(named, [len(pages)]) + self.assertIn("legal notices", pages[-1]) class TermGuardTests(BasicReportTestCase): @@ -623,19 +819,6 @@ def test_the_guard_finds_what_it_should(self): self.assertEqual(find_terms("an ordinary sentence about cloud exit"), []) self.assertEqual(find_terms("ICTs and DORAs"), []) - def test_the_platforms_are_named_only_on_the_last_two_pages(self): - _path, _context, reader = self.render(key="default") - pages = page_texts(reader) - - for text in pages[:-2]: - self.assertNotIn("escapecloud", text.lower()) - self.assertNotIn("exitcloud", text.lower()) - # The closing page keeps both platform URLs; the notices name EscapeCloud - self.assertIn("https://escapecloud.io", pages[-2]) - self.assertIn("https://exitcloud.io", pages[-2]) - self.assertIn("EscapeCloud", pages[-1]) - self.assertNotIn("exitcloud", pages[-1].lower()) - class LegalNoticesTests(BasicReportTestCase): ICON_MARKERS = {1: "Microsoft group", 2: "Amazon.com"} @@ -685,6 +868,21 @@ def test_the_icon_links_follow_the_provider(self): self.assertIn(AZURE_ICONS_URL, links) self.assertNotIn(AWS_ICONS_URL, links) + def test_the_icon_names_are_followed_by_their_addresses(self): + expected = { + 2: [ + f"AWS Architecture Icons ({AWS_ICONS_URL})", + f"AWS Trademark Guidelines ({AWS_TRADEMARK_URL})", + ], + 1: [f"Azure Architecture Icons ({AZURE_ICONS_URL})"], + } + for name, fixture, overrides in self.scenarios(): + with self.subTest(name): + context, _reader, pages = self.notices_page(name, fixture, **overrides) + last = pages[-1].replace(" ", "") + for text in expected[context.cloud_service_provider]: + self.assertIn(text.replace(" ", ""), last) + def test_the_cover_names_the_edition_and_no_classification(self): _context, _reader, pages = self.notices_page("offline", provenance=provenance()) cover = pages[0] diff --git a/tests/report/test_html_report.py b/tests/report/test_html_report.py index 128569d..de594af 100644 --- a/tests/report/test_html_report.py +++ b/tests/report/test_html_report.py @@ -11,7 +11,12 @@ from pypdf import PdfReader from cloudexit.engine import generate_report, perform_risk_assessment -from cloudexit.report.sections import AWS_ICONS_URL, AZURE_ICONS_URL, REPOSITORY_URL +from cloudexit.report.sections import ( + AWS_ICONS_URL, + AWS_TRADEMARK_URL, + AZURE_ICONS_URL, + REPOSITORY_URL, +) from cloudexit.report.wording import BLOCKS from cloudexit.resources import asset_path from tests.report.test_basic_report import page_texts @@ -227,9 +232,8 @@ def test_the_uploaded_chip_and_link_appear_only_on_uploaded_runs(self): chip = re.search(r']*>{name}', page + ) + ), + 1, + name, + ) + self.assertNotIn(f"({name})", text) + self.assertNotIn(f"( {name} )", text) + self.assertNotIn(f"{name} (", text) + for other in names.keys() - {scenario.provider}: + for name in names[other]: + self.assertNotIn(name, text) + def test_the_figures_notice_follows_the_state(self): def figures(case): page = self.page(case) diff --git a/tests/report/test_report_flow.py b/tests/report/test_report_flow.py index 1990737..6277a8c 100644 --- a/tests/report/test_report_flow.py +++ b/tests/report/test_report_flow.py @@ -5,6 +5,7 @@ import sqlite3 import tempfile import unittest +from dataclasses import replace from contextlib import nullcontext from pathlib import Path from unittest.mock import patch @@ -23,6 +24,7 @@ upload, write_upload_record, ) +from tests.support.html import html_text from tests.support.isolation import forget_cache, isolated_environ from tests.support.api import ( ASSESSMENT_ID, @@ -184,7 +186,7 @@ def test_an_offline_report_has_no_upload_trace(self): sorted(int(entry["id"]) for entry in document["data"]["risk_inventory"]), [2, 4], ) - self.assertNotIn("Not uploaded", self.html(report)) + self.assertNotIn("Upload failed", self.html(report)) self.assertTrue(Path(result["reports"]["PDF"]).is_file()) def generate(self, report, upload_info): @@ -207,10 +209,29 @@ def test_a_failed_upload_puts_a_banner_in_every_report(self): result = self.generate(report, info) self.assertTrue(result["success"], result.get("logs")) - self.assertIn("Not uploaded: upload_unavailable", self.html(report)) + self.assertIn("Upload failed: upload_unavailable.", self.html(report)) self.assertEqual(self.json_report(report)["meta"]["upload"], info) self.assertTrue(Path(result["reports"]["PDF"]).is_file()) + def test_a_failed_upload_that_was_kept_says_so_in_the_provenance(self): + report = self.report() + perform_risk_assessment(1, report) + info = { + "status": "failed", + "reason": REASON_UNAVAILABLE, + "sync_id": SYNC_ID, + "spooled": True, + } + + result = self.generate(report, info) + + self.assertTrue(result["success"], result.get("logs")) + self.assertIn( + "Upload failed: upload_unavailable. The run was kept for a later upload.", + html_text(self.html(report)), + ) + self.assertEqual(self.json_report(report)["meta"]["upload"], info) + def test_a_successful_upload_records_only_the_link(self): report = self.report() perform_risk_assessment(1, report) @@ -347,6 +368,17 @@ def test_report_info_and_the_record_file(self): self.assertIsNone(report_info(None)) self.assertIsNone(report_info(UploadResult(False, False))) + kept = replace(failed, spooled=True) + self.assertEqual( + report_info(kept), + { + "status": "failed", + "reason": REASON_REJECTED, + "sync_id": SYNC_ID, + "spooled": True, + }, + ) + detailed = UploadResult( True, True, @@ -547,7 +579,7 @@ def test_a_failed_upload_still_writes_the_report_and_exits_with_an_upload_code( self.assertEqual(ctx.exception.code, exit_code) self.assert_local_report(report) - self.assertIn(f"Not uploaded: {reason}", self.html(report)) + self.assertIn(f"Upload failed: {reason}.", self.html(report)) self.assertEqual( self.json_report(report)["meta"]["upload"]["reason"], reason ) @@ -650,7 +682,7 @@ def test_a_key_that_cannot_be_right_is_a_failed_upload_not_an_offline_run(self): mock_upload.assert_not_called() self.assertEqual(ctx.exception.code, codes.UPLOAD_AUTH) self.assert_local_report(report) - self.assertIn("Not uploaded: upload_auth", self.html(report)) + self.assertIn("Upload failed: upload_auth.", self.html(report)) # A configured platform keeps the requested depth: nothing forces Basic self.assertEqual(self.sync(report)["assessment_type"], 2) messages = [args[0] for args, _ in self.steps] @@ -691,7 +723,7 @@ def test_collection_problems_win_over_upload_codes(self): self.assertEqual(ctx.exception.code, codes.EGRESS) self.assert_local_report(report) - self.assertIn("Not uploaded", self.html(report)) + self.assertIn("Upload failed", self.html(report)) def test_a_sync_document_problem_wins_over_an_upload_code(self): report = self.report() diff --git a/tests/samples/render.py b/tests/samples/render.py index de5d4b5..84eaaaf 100644 --- a/tests/samples/render.py +++ b/tests/samples/render.py @@ -1,6 +1,7 @@ import logging import shutil import sqlite3 +import tempfile from contextlib import ExitStack from dataclasses import dataclass, field from datetime import datetime @@ -228,6 +229,19 @@ def _render(scenario, directory, catalogue_path): raise RenderError(f"egress.pdf failed: {report['egress_pdf_logs']}") +def _legacy_copy(catalogue_path, directory): + # The dataset of an older release: the same tables, no catalogue_meta + copy = Path(directory) / "legacy_catalogue.db" + shutil.copy(catalogue_path, copy) + conn = sqlite3.connect(copy) + try: + conn.execute("DROP TABLE IF EXISTS catalogue_meta") + conn.commit() + finally: + conn.close() + return copy + + def render_case(scenario, out_dir, *, catalogue_path): directory = Path(out_dir) / scenario.case if directory.exists(): @@ -237,8 +251,11 @@ def render_case(scenario, out_dir, *, catalogue_path): previous = store._active_override clock = frozen_datetime(scenario.clock) try: - store.set_active_catalogue(catalogue_path) with ExitStack() as stack: + if scenario.catalogue == "legacy": + scratch = stack.enter_context(tempfile.TemporaryDirectory()) + catalogue_path = _legacy_copy(catalogue_path, scratch) + store.set_active_catalogue(catalogue_path) # The warnings of the pipeline are about the sample data, not news stack.callback(logging.disable, logging.root.manager.disable) logging.disable(logging.WARNING) diff --git a/tests/samples/scenarios.py b/tests/samples/scenarios.py index 94a2f46..a531f25 100644 --- a/tests/samples/scenarios.py +++ b/tests/samples/scenarios.py @@ -23,11 +23,14 @@ "clock", "run_id", ) +OPTIONAL_FIELDS = ("catalogue",) +CATALOGUES = ("legacy",) SCOPE_FIELDS = { "aws": ("accountId", "region"), "azure": ("tenantId", "subscriptionId", "resourceGroupName"), } -UPLOAD_FIELDS = ("status", "url", "assessment_id", "sync_id", "reason") +UPLOAD_FIELDS = ("status", "url", "assessment_id", "sync_id", "reason", "spooled") +UPLOAD_STATUSES = ("uploaded", "failed") MONTH = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") CURRENCY = re.compile(r"^[A-Z]{3}$") UUID7 = re.compile( @@ -54,6 +57,8 @@ class Scenario: upload: dict | None clock: datetime run_id: str + # "legacy": a catalogue without catalogue_meta + catalogue: str | None = None @property def csp(self): @@ -218,10 +223,22 @@ def _upload(path, upload): if not isinstance(upload, dict): _fail(path, "upload", "expected an object or null") _only(path, "upload", upload, UPLOAD_FIELDS) - _text(path, "upload.status", upload.get("status")) + status = _text(path, "upload.status", upload.get("status")) + if status not in UPLOAD_STATUSES: + _fail(path, "upload.status", f"expected one of {list(UPLOAD_STATUSES)}") + if "reason" in upload: + _text(path, "upload.reason", upload["reason"]) + if "spooled" in upload and not isinstance(upload["spooled"], bool): + _fail(path, "upload.spooled", "expected true or false") return upload +def _catalogue(path, value): + if value is not None and value not in CATALOGUES: + _fail(path, "catalogue", f"expected one of {list(CATALOGUES)}, got {value!r}") + return value + + def _clock(path, value): text = _text(path, "clock", value) if not text.endswith("Z"): @@ -243,7 +260,7 @@ def load_scenario(path): for field in FIELDS: if field not in data: _fail(path, field, "missing") - _only(path, "(file)", data, FIELDS) + _only(path, "(file)", data, FIELDS + OPTIONAL_FIELDS) provider = data["provider"] if provider not in PROVIDER_IDS: @@ -280,6 +297,7 @@ def load_scenario(path): upload=_upload(path, data["upload"]), clock=_clock(path, data["clock"]), run_id=run_id, + catalogue=_catalogue(path, data.get("catalogue")), ) diff --git a/tests/samples/test_samples.py b/tests/samples/test_samples.py index 8026ed3..26524b6 100644 --- a/tests/samples/test_samples.py +++ b/tests/samples/test_samples.py @@ -224,35 +224,54 @@ def test_the_egress_notices_carry_the_egress_paragraph(self): class UploadTests(RenderedTestCase): - def test_uploaded_cases_show_their_link_or_assessment_id(self): + def test_uploaded_cases_show_their_link_or_assessment_id_on_page_two_only(self): linked = STATE["scenarios"]["azure_uploaded"].upload["url"] for pdf in ("report.pdf", "egress.pdf"): pages = self.pdf_pages("azure_uploaded", pdf) self.assertIn(linked, squashed(pages[1])) self.assertIn("Uploaded to the platform", pages[1]) - self.assertIn( - "The Full Assessment", self.pdf_pages("azure_uploaded", "report.pdf")[-2] - ) - self.assertIn( - linked, squashed(self.pdf_pages("azure_uploaded", "egress.pdf")[-2]) + if pdf == "report.pdf": + self.assertEqual(sum(linked in squashed(text) for text in pages), 1) + self.assertNotIn( + "The Full Assessment", + " ".join(self.pdf_pages("azure_uploaded", "report.pdf")), ) pages = self.pdf_pages("uploaded_no_link", "report.pdf") self.assertIn("Uploaded to the platform", pages[1]) - self.assertIn("Assessment ID: sample-assessment-0001", pages[-2]) + self.assertIn("Assessment ID: sample-assessment-0001", pages[1]) + self.assertEqual(sum("sample-assessment-0001" in text for text in pages), 1) self.assertNotIn("https://platform", " ".join(pages)) - def test_offline_cases_show_the_self_collected_line(self): + def test_offline_cases_show_the_self_collected_line_and_no_closing_page(self): for case, scenario in self.cases(): if scenario.upload is None: with self.subTest(case): pages = self.pdf_pages(case, "report.pdf") self.assertIn("Self-collected data", pages[1]) - self.assertIn("What the Full Assessment Adds", " ".join(pages)) + self.assertNotIn("What the Full Assessment Adds", " ".join(pages)) + self.assertIn("Legal Notices", pages[-1][:200]) if scenario.egress is not None: egress = self.pdf_pages(case, "egress.pdf") self.assertIn("Self-collected data", egress[1]) + def test_a_failed_upload_says_what_happened_in_both_pdfs_and_the_html(self): + wording = ( + "Upload failed: The platform did not answer in time. " + "The run was kept for a later upload." + ) + for pdf in ("report.pdf", "egress.pdf"): + if (self.folder("upload_failed") / pdf).is_file(): + self.assertIn(wording, self.pdf_pages("upload_failed", pdf)[1]) + pages = self.pdf_pages("upload_failed", "report.pdf") + self.assertNotIn("Self-collected data", " ".join(pages)) + self.assertIn(wording, html_text(self.text("upload_failed", "index.html"))) + chip = re.search( + r'