diff --git a/cloudexit/engine.py b/cloudexit/engine.py
index 5507910..15a4390 100644
--- a/cloudexit/engine.py
+++ b/cloudexit/engine.py
@@ -415,15 +415,19 @@ def perform_risk_assessment(exit_strategy: int, report_path: str) -> dict[str, A
return {"success": False, "logs": str(e)}
-def _load_tfstate_scope(raw_data_path: str) -> dict[str, Any] | None:
- # A missing or unreadable manifest must not fail report generation.
+def _load_tfstate_scope(
+ raw_data_path: str,
+) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+ # A missing or unreadable manifest must not fail report generation. Older
+ # manifests have no coverage.
manifest_path = os.path.join(raw_data_path, "tfstate_manifest.json")
try:
with open(manifest_path, "r", encoding="utf-8") as manifest_file:
- return json.load(manifest_file).get("scope")
+ manifest = json.load(manifest_file)
+ return manifest.get("scope"), manifest.get("coverage")
except (OSError, json.JSONDecodeError) as e:
logger.warning("Could not read the tfstate manifest for report scope: %s", e)
- return None
+ return None, None
# Stage 8
@@ -477,10 +481,10 @@ def generate_report(
reports = {}
# One context feeds all three reports, so they say the same things
- tfstate_scope = (
+ tfstate_scope, tfstate_coverage = (
_load_tfstate_scope(raw_data_path)
if provider_details.get("tfstatePath")
- else None
+ else (None, None)
)
provenance = collect_provenance(db_path, run_id=sync_id, upload=upload)
context = build_context(
@@ -496,6 +500,7 @@ def generate_report(
alternative_technology_organizations=alternative_technology_organizations,
exit_strategy=exit_strategy,
tfstate_scope=tfstate_scope,
+ tfstate_coverage=tfstate_coverage,
provenance=provenance,
report_path=report_path,
)
diff --git a/cloudexit/platform/upload.py b/cloudexit/platform/upload.py
index 33ac4c1..c63ee36 100644
--- a/cloudexit/platform/upload.py
+++ b/cloudexit/platform/upload.py
@@ -250,7 +250,7 @@ def read_sync_files(
return found, problems
-REPORT_KEYS = ("status", "reason", "sync_id", "assessment_id", "url")
+REPORT_KEYS = ("status", "reason", "sync_id", "assessment_id", "url", "spooled")
def _record(result: UploadResult | None) -> dict[str, Any] | None:
@@ -273,9 +273,9 @@ def _record(result: UploadResult | None) -> dict[str, Any] | None:
return {key: value for key, value in info.items() if value}
-def report_info(result: UploadResult | None) -> dict[str, str] | None:
- # What the local report may show: the status and the link, nothing the
- # platform computed.
+def report_info(result: UploadResult | None) -> dict[str, Any] | None:
+ # What the local report may show: the status, the link and whether a failed
+ # run was kept, nothing the platform computed.
record = _record(result)
if record is None:
return None
diff --git a/cloudexit/report/context.py b/cloudexit/report/context.py
index da43ca6..2de9e54 100644
--- a/cloudexit/report/context.py
+++ b/cloudexit/report/context.py
@@ -38,8 +38,10 @@ class Provenance:
uploaded: bool = False
platform_url: str | None = None
assessment_id: str | None = None
- # Why a requested upload did not happen, when it did not
+ # A requested upload that failed, why, and whether the run was kept
+ upload_failed: bool = False
upload_reason: str | None = None
+ upload_spooled: bool = False
@property
def run_short_id(self):
@@ -79,9 +81,11 @@ def collect_provenance(db_path, *, run_id=None, upload=None, collected_at=None):
uploaded=upload.get("status") == "uploaded",
platform_url=upload.get("url") if upload.get("status") == "uploaded" else None,
assessment_id=upload.get("assessment_id"),
+ upload_failed=upload.get("status") == "failed",
upload_reason=(
- upload.get("reason") if upload.get("status") != "uploaded" else None
+ upload.get("reason") if upload.get("status") == "failed" else None
),
+ upload_spooled=bool(upload.get("spooled")) and upload.get("status") == "failed",
)
@@ -147,6 +151,14 @@ class ScopeFacts:
resource_group: str = ""
file_name: str = ""
locations: tuple = ()
+ # State file instances left out: of other providers, of types not in the
+ # data catalogue
+ not_assessed_other: int = 0
+ not_assessed_unmapped: int = 0
+
+ @property
+ def not_assessed(self):
+ return self.not_assessed_other + self.not_assessed_unmapped
@dataclass
@@ -316,7 +328,14 @@ def _alternative_entries(
return entries
-def scope_facts(provider_details, cloud_service_provider, tfstate_scope):
+def _excluded(coverage, key):
+ value = (coverage or {}).get(key)
+ return value if isinstance(value, int) and value > 0 else 0
+
+
+def scope_facts(
+ provider_details, cloud_service_provider, tfstate_scope, tfstate_coverage=None
+):
provider = PROVIDER_NAMES.get(cloud_service_provider, "–")
if provider_details.get("tfstatePath"):
scope = tfstate_scope or {
@@ -343,6 +362,12 @@ def scope_facts(provider_details, cloud_service_provider, tfstate_scope):
tuple(rows),
file_name=scope.get("file") or "",
locations=locations,
+ not_assessed_other=_excluded(
+ tfstate_coverage, "instances_excluded_other_provider"
+ ),
+ not_assessed_unmapped=_excluded(
+ tfstate_coverage, "instances_excluded_unmapped"
+ ),
)
if cloud_service_provider == 1:
rows = (
@@ -379,6 +404,7 @@ def build_context(
alternative_technology_organizations: list[dict[str, Any]],
exit_strategy: int,
tfstate_scope: dict[str, Any] | None,
+ tfstate_coverage: dict[str, Any] | None = None,
provenance: Provenance,
report_path: str,
) -> ReportContext:
@@ -392,7 +418,7 @@ def build_context(
provider=PROVIDER_NAMES.get(csp, "–"),
exit_strategy=exit_strategy,
strategy_code=STRATEGY_CODES.get(int(exit_strategy), ""),
- scope=scope_facts(provider_details, csp, tfstate_scope),
+ scope=scope_facts(provider_details, csp, tfstate_scope, tfstate_coverage),
provenance=provenance,
resources=resources,
costs=_cost_entries(cost_data, provenance.collected_at),
diff --git a/cloudexit/report/evidence.py b/cloudexit/report/evidence.py
index e0ea97e..7c81168 100644
--- a/cloudexit/report/evidence.py
+++ b/cloudexit/report/evidence.py
@@ -53,6 +53,22 @@ def build_document_table(
return key_value_table(rows, context.appearance)
+def _not_assessed_text(scope, b):
+ parts = [
+ _counted(b, key, n)
+ for key, n in (
+ ("ev_not_assessed_other", scope.not_assessed_other),
+ ("ev_not_assessed_unmapped", scope.not_assessed_unmapped),
+ )
+ if n
+ ]
+ return ", ".join(parts)
+
+
+def _counted(b, key, n):
+ return b[f"{key}_one"] if n == 1 else b[f"{key}_many"].format(n=n)
+
+
def scope_rows(context, b):
# (label, text, monospaced), the same for the PDF and the HTML report
scope = context.scope
@@ -91,6 +107,8 @@ def scope_rows(context, b):
):
if key in values:
rows.append((b[label], values[key] or b["ev_not_determinable"], False))
+ if scope.not_assessed:
+ rows.append((b["ev_not_assessed"], _not_assessed_text(scope, b), False))
source = b["ev_source_tfstate" if scope.kind == "tfstate" else "ev_source_live"]
rows.append(
(
@@ -117,6 +135,19 @@ def link_markup(url, color):
return f'{escape(url)}'
+def platform_failure(provenance, b):
+ # What happened to an upload that failed, as it was true when the report was written
+ if provenance.upload_reason:
+ text = b["ev_platform_failed"].format(
+ reason=provenance.upload_reason.rstrip(". ")
+ )
+ else:
+ text = b["ev_platform_failed_no_reason"]
+ if provenance.upload_spooled:
+ text = f"{text} {b['ev_platform_kept']}"
+ return text
+
+
def build_provenance_table(context, b):
provenance = context.provenance
if provenance.catalogue_version and provenance.catalogue_sha256:
@@ -137,7 +168,11 @@ def build_provenance_table(context, b):
text = escape(b["ev_platform_uploaded"])
if provenance.platform_url:
text = f"{text}: {link_markup(provenance.platform_url, context.appearance.heading2_color)}"
+ elif provenance.assessment_id:
+ text = f"{text} · {escape(b['fa_uploaded_id'].format(assessment_id=provenance.assessment_id))}"
rows.append((b["ev_platform_label"], Paragraph(text, KV_VALUE)))
+ elif provenance.upload_failed:
+ rows.append((b["ev_platform_label"], platform_failure(provenance, b)))
else:
rows.append((b["ev_platform_label"], b["ev_platform_none"]))
return key_value_table(rows, context.appearance)
diff --git a/cloudexit/report/executive_summary.py b/cloudexit/report/executive_summary.py
index eb1b4d5..b36e939 100644
--- a/cloudexit/report/executive_summary.py
+++ b/cloudexit/report/executive_summary.py
@@ -14,7 +14,6 @@
from .sections import (
alternative_technologies,
costs_overview,
- full_assessment,
glossary,
introduction,
resource_inventory,
@@ -45,8 +44,6 @@ def build_content(context, styles, b):
content.append(Spacer(1, 18))
content += glossary(context, styles, b)
content.append(chapter_break())
- content += full_assessment(context, styles, b)
- content.append(chapter_break())
content += legal_notices(context, styles, b)
return content
diff --git a/cloudexit/report/html_sections.py b/cloudexit/report/html_sections.py
index e41ef37..9cd6146 100644
--- a/cloudexit/report/html_sections.py
+++ b/cloudexit/report/html_sections.py
@@ -1,6 +1,6 @@
from markupsafe import Markup
-from .evidence import scope_rows
+from .evidence import platform_failure, scope_rows
from .helpers import (
format_change,
initials,
@@ -11,12 +11,9 @@
strategy_name,
)
from .html_egress import egress_page
-from .notices import text_parts
+from .notices import icon_notice, text_parts
from .palette import SEVERITIES
from .sections import (
- AWS_ICONS_URL,
- AWS_TRADEMARK_URL,
- AZURE_ICONS_URL,
PRODUCT_SITES,
REPOSITORY_URL,
)
@@ -406,6 +403,15 @@ def _inventory(context, b):
]
if scope.kind == "tfstate":
caption_parts.append(b["html_inv_not_managed"])
+ if scope.not_assessed:
+ key = (
+ "html_inv_not_assessed_one"
+ if scope.not_assessed == 1
+ else "html_inv_not_assessed_many"
+ )
+ caption_parts.append(
+ b[key].format(n=scope.not_assessed, about=b["html_nav_about"])
+ )
if scope.kind == "azure":
empty = b["html_inv_empty_resource_group"].format(
@@ -544,10 +550,8 @@ def _count_more(b, n):
def _platform_cell(context, b):
provenance = context.provenance
if not provenance.uploaded:
- if provenance.upload_reason:
- return [
- b["html_platform_not_uploaded"].format(reason=provenance.upload_reason)
- ]
+ if provenance.upload_failed:
+ return [platform_failure(provenance, b)]
return [b["ev_platform_none"]]
parts = [b["ev_platform_uploaded"]]
if provenance.platform_url:
@@ -667,20 +671,7 @@ def _about(context, egress, b):
def _notices(context, egress, b):
provenance = context.provenance
- icons = None
- if context.cloud_service_provider == 2:
- icons = text_parts(
- b["notices_icons_aws"],
- {
- "icons_url": (b["html_label_aws_icons"], AWS_ICONS_URL),
- "guidelines_url": (b["html_label_aws_guidelines"], AWS_TRADEMARK_URL),
- },
- )
- elif context.cloud_service_provider == 1:
- icons = text_parts(
- b["notices_icons_azure"],
- {"icons_url": (b["html_label_azure_icons"], AZURE_ICONS_URL)},
- )
+ icons = icon_notice(context.cloud_service_provider, b)
if context.scope.kind == "tfstate":
figures = b["html_figures_tfstate"]
else:
diff --git a/cloudexit/report/notices.py b/cloudexit/report/notices.py
index 60c96d7..2f81551 100644
--- a/cloudexit/report/notices.py
+++ b/cloudexit/report/notices.py
@@ -37,14 +37,22 @@ def text_parts(template, links, values=None):
return parts
-def _icon_notice(provider, b):
+def icon_notice(provider, b):
+ # The names are the links: the HTML shows the name, the PDF the name and, in
+ # brackets, the address
if provider == 2:
return text_parts(
b["notices_icons_aws"],
- {"icons_url": AWS_ICONS_URL, "guidelines_url": AWS_TRADEMARK_URL},
+ {
+ "icons": (b["notices_label_aws_icons"], AWS_ICONS_URL),
+ "guidelines": (b["notices_label_aws_guidelines"], AWS_TRADEMARK_URL),
+ },
)
if provider == 1:
- return text_parts(b["notices_icons_azure"], {"icons_url": AZURE_ICONS_URL})
+ return text_parts(
+ b["notices_icons_azure"],
+ {"icons": (b["notices_label_azure_icons"], AZURE_ICONS_URL)},
+ )
return None
@@ -60,7 +68,7 @@ def notices_html(provider, generated_at, b=BLOCKS, egress=False):
*([[b["notices_egress"]]] if egress else []),
]
trademarks = [[b["notices_trademark_own"]], [b["notices_trademark_others"]]]
- icons = _icon_notice(provider, b)
+ icons = icon_notice(provider, b)
if icons:
trademarks.append(icons)
return {
@@ -85,9 +93,20 @@ def notices_html(provider, generated_at, b=BLOCKS, egress=False):
}
+def _link_markup(label, url, color):
+ # A link that is not its own address prints the address too, for print
+ if label == url:
+ return link_markup(url, color)
+ return f"{escape(label)} ({link_markup(url, color)})"
+
+
def parts_markup(parts, color):
return "".join(
- link_markup(part[1], color) if isinstance(part, tuple) else escape(part)
+ (
+ _link_markup(part[0], part[1], color)
+ if isinstance(part, tuple)
+ else escape(part)
+ )
for part in parts
)
diff --git a/cloudexit/report/sections.py b/cloudexit/report/sections.py
index c9d5584..5df8c54 100644
--- a/cloudexit/report/sections.py
+++ b/cloudexit/report/sections.py
@@ -7,7 +7,6 @@
from reportlab.pdfbase.pdfmetrics import stringWidth
from reportlab.platypus import (
Flowable,
- KeepTogether,
ListFlowable,
ListItem,
Paragraph,
@@ -17,7 +16,6 @@
)
from .brief_summary import cost_finding
-from .evidence import link_markup
from .flow import section
from .fonts import FONT, FONT_BOLD, FONT_MONO
from .placeholder import LabelWithPlaceholder
@@ -56,9 +54,6 @@
"RiskCode", fontName=FONT_MONO
)
RISK_MUTED_RIGHT = text_style("RiskMutedRight", 8.5, 11, MUTED, alignment=TA_RIGHT)
-COMMAND = text_style("CommandLine", 8.5, 12, INK).clone(
- "CommandLine", fontName=FONT_MONO, leftIndent=8, spaceBefore=4, spaceAfter=8
-)
GLOSSARY_TERM_WIDTH = 4.2 * cm
RISK_BAR_MIN_SEGMENT = 62
ALT_EXAMPLES = 3
@@ -358,6 +353,11 @@ def resource_inventory(context, styles, b):
("NOSPLIT", (0, total_row - 1), (-1, total_row)),
],
)
+ caption = b["es_res_caption"]
+ skipped = context.scope.not_assessed
+ if skipped:
+ key = "res_not_assessed_one" if skipped == 1 else "res_not_assessed_many"
+ caption = f"{caption} {b[key].format(n=skipped)}"
return section(
head,
[
@@ -371,7 +371,7 @@ def resource_inventory(context, styles, b):
Spacer(1, 12),
table,
Spacer(1, 6),
- Paragraph(b["es_res_caption"], ALT_CAPTION),
+ Paragraph(caption, ALT_CAPTION),
],
)
@@ -576,11 +576,7 @@ def risk_inventory(context, styles, b):
def _examples_cell(entry, b):
cell = []
for alt in entry.alternatives[:ALT_EXAMPLES]:
- vendor = (
- f' · {escape(alt.vendor)}'
- if alt.vendor
- else ""
- )
+ vendor = f' · {escape(alt.vendor or b["ev_dash"])}'
cell.append(Paragraph(f"{escape(alt.name)}{vendor}", ALT_BODY))
flags = [
b[key]
@@ -758,76 +754,3 @@ def glossary(context, styles, b):
Spacer(1, 12),
table,
]
-
-
-# Closing page
-def full_assessment(context, styles, b):
- body = styles["ContentStyle"]
- provenance = context.provenance
- if provenance.uploaded:
- content = [
- Paragraph(b["fa_heading_uploaded"], styles["Heading1"]),
- Paragraph(escape(b["fa_uploaded"]), body),
- ]
- if provenance.platform_url:
- link = link_markup(
- provenance.platform_url, context.appearance.heading2_color
- )
- content.append(Paragraph(b["fa_uploaded_link"].format(link=link), body))
- elif provenance.assessment_id:
- content.append(
- Paragraph(
- escape(
- b["fa_uploaded_id"].format(
- assessment_id=provenance.assessment_id
- )
- ),
- body,
- )
- )
- return content
-
- rows = [[_text(term, ALT_BODY_BOLD), _text(text)] for term, text in b["fa_items"]]
- table = Table(
- rows,
- colWidths=[GLOSSARY_TERM_WIDTH, 16 * cm - GLOSSARY_TERM_WIDTH],
- hAlign="LEFT",
- )
- table.setStyle(
- TableStyle(
- [
- (
- "LINEABOVE",
- (0, 0),
- (-1, 0),
- 1,
- HexColor(context.appearance.header_row_color),
- ),
- ("LINEBELOW", (0, 0), (-1, -1), 0.4, HexColor(HAIRLINE)),
- ("LEFTPADDING", (0, 0), (-1, -1), 5),
- ("RIGHTPADDING", (0, 0), (-1, -1), 5),
- ("TOPPADDING", (0, 0), (-1, -1), 5),
- ("BOTTOMPADDING", (0, 0), (-1, -1), 5),
- ("VALIGN", (0, 0), (-1, -1), "TOP"),
- ]
- )
- )
- color = context.appearance.heading2_color
- sites = b["fa_sites"].format(
- **{name: link_markup(url, color) for name, url in PRODUCT_SITES}
- )
- return [
- Paragraph(b["fa_heading"], styles["Heading1"]),
- Paragraph(escape(b["fa_lead"]), body),
- Spacer(1, 6),
- table,
- Spacer(1, 6),
- KeepTogether(
- [
- Paragraph(b["fa_how_heading"], styles["Heading2"]),
- Paragraph(escape(b["fa_how"]), body),
- Paragraph(escape(b["fa_command"]), COMMAND),
- Paragraph(sites, body),
- ]
- ),
- ]
diff --git a/cloudexit/report/wording.py b/cloudexit/report/wording.py
index 86cc08f..d54befd 100644
--- a/cloudexit/report/wording.py
+++ b/cloudexit/report/wording.py
@@ -27,6 +27,25 @@
"ev_resource_groups": "Resource Groups",
"ev_regions": "Regions",
"ev_not_determinable": "Not determinable from the state",
+ "ev_not_assessed": "Not Assessed",
+ "ev_not_assessed_other_one": "1 resource of another provider",
+ "ev_not_assessed_other_many": "{n} resources of other providers",
+ "ev_not_assessed_unmapped_one": "1 resource of a type not in the data catalogue",
+ "ev_not_assessed_unmapped_many": (
+ "{n} resources of types not in the data catalogue"
+ ),
+ "res_not_assessed_one": (
+ "1 resource in the state file was not assessed (see page 2)."
+ ),
+ "res_not_assessed_many": (
+ "{n} resources in the state file were not assessed (see page 2)."
+ ),
+ "html_inv_not_assessed_one": (
+ "1 resource in the state file was not assessed (see {about})"
+ ),
+ "html_inv_not_assessed_many": (
+ "{n} resources in the state file were not assessed (see {about})"
+ ),
"ev_source_label": "Source",
"ev_source_live": "Provider APIs",
"ev_source_tfstate": "Terraform State",
@@ -38,6 +57,9 @@
"ev_platform_label": "Platform",
"ev_platform_none": "Self-collected data: no platform evaluated this report.",
"ev_platform_uploaded": "Uploaded to the platform",
+ "ev_platform_failed": "Upload failed: {reason}.",
+ "ev_platform_failed_no_reason": "Upload failed.",
+ "ev_platform_kept": "The run was kept for a later upload.",
"ev_stored_footnote": (
"This document was generated when run {run} completed and is stored "
"unchanged. Later runs are reported in their own documents."
@@ -335,41 +357,6 @@
"definition": "One completed assessment of the defined scope, identified by the run ID on page 2.",
},
],
- "fa_heading": "What the Full Assessment Adds",
- "fa_lead": (
- "This Executive Summary is the Basic assessment. It was collected and "
- "evaluated locally and covers the resource inventory, "
- "costs, exit risks and alternative technologies of the defined scope. "
- "The full assessment on a platform adds:"
- ),
- "fa_items": [
- (
- "Exit Readiness Grade",
- "a letter grade from A to F that shows how ready the defined scope is for an exit.",
- ),
- (
- "Vendor Lock-in Score",
- "a rating of how easily each resource type can be replaced, in terms of skills, technology and operations.",
- ),
- (
- "Scored Alternatives",
- "the alternative technologies for each resource type, rated and ranked.",
- ),
- ],
- "fa_how_heading": "How to Get Them",
- "fa_how": (
- "Connect cloudexit to a platform with an API key and run the "
- "assessment again; the results are uploaded and evaluated there. For "
- "example:"
- ),
- "fa_command": "cloudexit aws --host HOST --api-key-file KEY_FILE",
- "fa_sites": "Platforms: {escapecloud} · {exitcloud}",
- "fa_heading_uploaded": "The Full Assessment",
- "fa_uploaded": (
- "This assessment was uploaded. The Exit Readiness Grade, the Vendor "
- "Lock-in Score, scored alternatives and the Egress & Data Landscape "
- "are available in its reports on the platform."
- ),
"fa_uploaded_link": "Open the reports: {link}",
"fa_uploaded_id": "Assessment ID: {assessment_id}",
"eg_heading_introduction": "Introduction",
@@ -652,20 +639,20 @@
"affiliation."
),
"notices_icons_aws": (
- "AWS service icons are taken from the AWS Architecture Icons "
- "({icons_url}); their use is subject to the terms published there and "
- "in the AWS Trademark Guidelines ({guidelines_url}). Amazon Web "
- "Services and AWS are trademarks of Amazon.com, Inc. or its "
- "affiliates. This report is not endorsed by or affiliated with Amazon "
- "Web Services."
+ "AWS service icons are taken from the {icons}; their use is subject to "
+ "the terms published there and in the {guidelines}. Amazon Web Services "
+ "and AWS are trademarks of Amazon.com, Inc. or its affiliates. This "
+ "report is not endorsed by or affiliated with Amazon Web Services."
),
"notices_icons_azure": (
- "Azure service icons are taken from the Azure Architecture Icons "
- "({icons_url}); their use is subject to Microsoft's icon terms "
- "published there. Microsoft and Azure are trademarks of the Microsoft "
- "group of companies. This report is not endorsed by or affiliated with "
- "Microsoft."
- ),
+ "Azure service icons are taken from the {icons}; their use is subject "
+ "to Microsoft's icon terms published there. Microsoft and Azure are "
+ "trademarks of the Microsoft group of companies. This report is not "
+ "endorsed by or affiliated with Microsoft."
+ ),
+ "notices_label_aws_icons": "AWS Architecture Icons",
+ "notices_label_aws_guidelines": "AWS Trademark Guidelines",
+ "notices_label_azure_icons": "Azure Architecture Icons",
"notices_software_heading": "Software",
"notices_software": (
"cloudexit is free software, licensed under the GNU Affero General "
@@ -752,7 +739,6 @@
"and operations, with the alternatives rated and ranked."
),
"html_scores_on_platform": "For this run it is on the platform.",
- "html_platform_not_uploaded": "Not uploaded: {reason}",
"html_hint_title": "EscapeCloud Light & Enterprise",
"html_hint_text": (
"EscapeCloud's scoring engine offers several scoring methods to assess "
@@ -934,9 +920,6 @@
"html_notice_advice": (
"It is not legal, financial or investment advice, an audit or an exit plan."
),
- "html_label_aws_icons": "AWS Architecture Icons",
- "html_label_aws_guidelines": "AWS Trademark Guidelines",
- "html_label_azure_icons": "Azure Architecture Icons",
"html_label_repository": "github.com/escapecloud/cloudexit",
"html_figures_heading": "Figures",
"html_figures": (
@@ -1049,6 +1032,7 @@
"ev_catalogue_label",
"ev_run_id",
"ev_platform_label",
+ "ev_not_assessed",
"cover_title_executive_summary",
"cover_title_egress",
"ev_egress_report",
@@ -1143,9 +1127,6 @@
"es_heading_severity",
"es_severity_th_meaning",
"es_heading_glossary",
- "fa_heading",
- "fa_how_heading",
- "fa_heading_uploaded",
"html_download",
"html_download_summary",
"html_download_egress",
diff --git a/tests/cli/test_upload_modes.py b/tests/cli/test_upload_modes.py
index f0b3299..7cd86f2 100644
--- a/tests/cli/test_upload_modes.py
+++ b/tests/cli/test_upload_modes.py
@@ -91,7 +91,7 @@ def test_auto_with_a_platform_that_refuses_the_key_writes_the_report_first(self)
self.assertEqual(ctx.exception.code, codes.UPLOAD_AUTH)
self.assert_local_report(report)
- self.assertIn("Not uploaded", self.html(report))
+ self.assertIn("Upload failed", self.html(report))
def test_required_without_a_platform_fails_before_collecting_anything(self):
report = self.report()
diff --git a/tests/fixtures/samples/azure_uploaded.json b/tests/fixtures/samples/azure_uploaded.json
index 6d65415..1131ce0 100644
--- a/tests/fixtures/samples/azure_uploaded.json
+++ b/tests/fixtures/samples/azure_uploaded.json
@@ -1,6 +1,6 @@
{
"name": "Sample – Document Archive",
- "purpose": "The same run, uploaded with a link: the uploaded closing page and the link on page 2 of both documents.",
+ "purpose": "Uploaded with a link: the link on page 2 of both PDFs and in the HTML.",
"provider": "azure",
"exit_strategy": 1,
"scope": {
diff --git a/tests/fixtures/samples/egress_unsized.json b/tests/fixtures/samples/egress_unsized.json
new file mode 100644
index 0000000..7c3c17e
--- /dev/null
+++ b/tests/fixtures/samples/egress_unsized.json
@@ -0,0 +1,83 @@
+{
+ "name": "Sample – Archive Migration Staging",
+ "purpose": "Egress where no data store has a size: fee and totals without a figure, every type not sized, no free-tier headroom.",
+ "provider": "azure",
+ "exit_strategy": 1,
+ "scope": {
+ "tenantId": "00000000-0000-4000-8000-0000000000a1",
+ "subscriptionId": "00000000-0000-4000-8000-0000000000a2",
+ "resourceGroupName": "rg-sample-staging"
+ },
+ "resources": [
+ {
+ "type": "Microsoft.Storage/storageAccounts",
+ "region": "westeurope",
+ "count": 3
+ },
+ {
+ "type": "Microsoft.Sql/servers/databases",
+ "region": "westeurope",
+ "count": 2
+ },
+ {
+ "type": "Microsoft.DocumentDB/databaseAccounts",
+ "region": "westeurope",
+ "count": 1
+ },
+ {
+ "type": "Microsoft.RecoveryServices/vaults",
+ "region": "westeurope",
+ "count": 1
+ }
+ ],
+ "costs": {
+ "currency": "EUR",
+ "months": {
+ "2026-06": 830.4,
+ "2026-07": 842.15,
+ "2026-08": 851.0,
+ "2026-09": 860.3,
+ "2026-10": 855.75,
+ "2026-11": 351.2
+ }
+ },
+ "egress": {
+ "archive_tiers": [
+ "Archive"
+ ],
+ "resources": [
+ {
+ "type": "Microsoft.Storage/storageAccounts",
+ "name": "samplestagingblobs",
+ "size_gib": null,
+ "unknown": true
+ },
+ {
+ "type": "Microsoft.Storage/storageAccounts",
+ "name": "samplestagingexports",
+ "size_gib": null,
+ "unknown": true
+ },
+ {
+ "type": "Microsoft.Sql/servers/databases",
+ "name": "sample-staging-orders",
+ "size_gib": null,
+ "unknown": true
+ },
+ {
+ "type": "Microsoft.DocumentDB/databaseAccounts",
+ "name": "sample-staging-search",
+ "size_gib": null,
+ "unknown": true
+ },
+ {
+ "type": "Microsoft.RecoveryServices/vaults",
+ "name": "sample-staging-vault",
+ "size_gib": null
+ }
+ ]
+ },
+ "upload": null,
+ "clock": "2026-11-12T09:30:00Z",
+ "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0014"
+}
diff --git a/tests/fixtures/samples/empty_with_costs.json b/tests/fixtures/samples/empty_with_costs.json
new file mode 100644
index 0000000..cadea8b
--- /dev/null
+++ b/tests/fixtures/samples/empty_with_costs.json
@@ -0,0 +1,27 @@
+{
+ "name": "Sample – Decommissioned Test Environment",
+ "purpose": "An empty resource group that still shows six months of small costs: empty states next to real figures.",
+ "provider": "azure",
+ "exit_strategy": 1,
+ "scope": {
+ "tenantId": "00000000-0000-4000-8000-0000000000a1",
+ "subscriptionId": "00000000-0000-4000-8000-0000000000a2",
+ "resourceGroupName": "rg-sample-decommissioned"
+ },
+ "resources": [],
+ "costs": {
+ "currency": "EUR",
+ "months": {
+ "2026-06": 38.2,
+ "2026-07": 31.75,
+ "2026-08": 24.4,
+ "2026-09": 12.9,
+ "2026-10": 6.35,
+ "2026-11": 1.8
+ }
+ },
+ "egress": null,
+ "upload": null,
+ "clock": "2026-11-12T09:30:00Z",
+ "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0015"
+}
diff --git a/tests/fixtures/samples/legacy_catalogue.json b/tests/fixtures/samples/legacy_catalogue.json
new file mode 100644
index 0000000..2254651
--- /dev/null
+++ b/tests/fixtures/samples/legacy_catalogue.json
@@ -0,0 +1,43 @@
+{
+ "name": "Sample – Internal Wiki",
+ "purpose": "A dataset without catalogue_meta: the catalogue version is unknown on page 2 and in About This Report.",
+ "provider": "aws",
+ "exit_strategy": 2,
+ "scope": {
+ "accountId": "123456789012",
+ "region": "eu-central-1"
+ },
+ "resources": [
+ {
+ "type": "AWS.ec2.describe_instances.Reservations",
+ "region": "eu-central-1",
+ "count": 2
+ },
+ {
+ "type": "AWS.s3.list_buckets.Buckets",
+ "region": "eu-central-1",
+ "count": 1
+ },
+ {
+ "type": "AWS.rds.describe_db_instances.DBInstances",
+ "region": "eu-central-1",
+ "count": 1
+ }
+ ],
+ "costs": {
+ "currency": "USD",
+ "months": {
+ "2026-06": 96.4,
+ "2026-07": 98.1,
+ "2026-08": 97.25,
+ "2026-09": 99.6,
+ "2026-10": 98.8,
+ "2026-11": 40.3
+ }
+ },
+ "egress": null,
+ "upload": null,
+ "clock": "2026-11-12T09:30:00Z",
+ "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0016",
+ "catalogue": "legacy"
+}
diff --git a/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate b/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate
new file mode 100644
index 0000000..2724066
--- /dev/null
+++ b/tests/fixtures/samples/tfstate/sample_aws_skipped.tfstate
@@ -0,0 +1,126 @@
+{
+ "version": 4,
+ "terraform_version": "1.14.6",
+ "serial": 31,
+ "lineage": "00000000-0000-4000-8000-0000000000c3",
+ "outputs": {},
+ "resources": [
+ {
+ "mode": "managed",
+ "type": "aws_instance",
+ "name": "app",
+ "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "availability_zone": "eu-central-1a",
+ "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef0"
+ }
+ },
+ {
+ "schema_version": 0,
+ "attributes": {
+ "availability_zone": "eu-central-1b",
+ "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef1"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "aws_s3_bucket",
+ "name": "assets",
+ "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "region": "eu-central-1",
+ "bucket": "sample-landing-assets",
+ "arn": "arn:aws:s3:::sample-landing-assets"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "aws_iam_role",
+ "name": "deploy",
+ "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "arn": "arn:aws:iam::123456789012:role/sample-deploy"
+ }
+ },
+ {
+ "schema_version": 0,
+ "attributes": {
+ "arn": "arn:aws:iam::123456789012:role/sample-readonly"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "aws_cloudwatch_log_group",
+ "name": "app",
+ "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "arn": "arn:aws:logs:eu-central-1:123456789012:log-group:/sample/app"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "azurerm_resource_group",
+ "name": "shared",
+ "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "location": "westeurope",
+ "name": "rg-sample-shared"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "azurerm_storage_account",
+ "name": "backup",
+ "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "location": "westeurope",
+ "name": "samplebackupstore"
+ }
+ }
+ ]
+ },
+ {
+ "mode": "managed",
+ "type": "google_storage_bucket",
+ "name": "exports",
+ "provider": "provider[\"registry.terraform.io/hashicorp/google\"]",
+ "instances": [
+ {
+ "schema_version": 0,
+ "attributes": {
+ "location": "EU",
+ "name": "sample-exports"
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/tests/fixtures/samples/tfstate_skipped.json b/tests/fixtures/samples/tfstate_skipped.json
new file mode 100644
index 0000000..c156841
--- /dev/null
+++ b/tests/fixtures/samples/tfstate_skipped.json
@@ -0,0 +1,15 @@
+{
+ "name": "Sample – Multi-Cloud Landing Zone (Terraform)",
+ "purpose": "A state file with instances of another provider and of types not in the data catalogue: the Not Assessed row on page 2 and in About This Report.",
+ "provider": "aws",
+ "exit_strategy": 1,
+ "scope": {
+ "tfstate": "sample_aws_skipped.tfstate"
+ },
+ "resources": [],
+ "costs": null,
+ "egress": null,
+ "upload": null,
+ "clock": "2026-11-12T09:30:00Z",
+ "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0017"
+}
diff --git a/tests/fixtures/samples/upload_failed.json b/tests/fixtures/samples/upload_failed.json
new file mode 100644
index 0000000..5ebd294
--- /dev/null
+++ b/tests/fixtures/samples/upload_failed.json
@@ -0,0 +1,52 @@
+{
+ "name": "Sample – Order Tracking Service",
+ "purpose": "A failed upload that was kept for a later upload: the wording on page 2 and in About This Report.",
+ "provider": "aws",
+ "exit_strategy": 1,
+ "scope": {
+ "accountId": "123456789012",
+ "region": "eu-central-1"
+ },
+ "resources": [
+ {
+ "type": "AWS.ec2.describe_instances.Reservations",
+ "region": "eu-central-1",
+ "count": 5
+ },
+ {
+ "type": "AWS.s3.list_buckets.Buckets",
+ "region": "eu-central-1",
+ "count": 3
+ },
+ {
+ "type": "AWS.rds.describe_db_instances.DBInstances",
+ "region": "eu-central-1",
+ "count": 1
+ },
+ {
+ "type": "AWS.lambda.list_functions.Functions",
+ "region": "eu-central-1",
+ "count": 6
+ }
+ ],
+ "costs": {
+ "currency": "USD",
+ "months": {
+ "2026-06": 412.5,
+ "2026-07": 398.2,
+ "2026-08": 405.75,
+ "2026-09": 421.1,
+ "2026-10": 418.9,
+ "2026-11": 171.6
+ }
+ },
+ "egress": null,
+ "upload": {
+ "status": "failed",
+ "reason": "The platform did not answer in time",
+ "spooled": true,
+ "sync_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0013"
+ },
+ "clock": "2026-11-12T09:30:00Z",
+ "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0013"
+}
diff --git a/tests/report/test_basic_report.py b/tests/report/test_basic_report.py
index 24606d8..64d3e4d 100644
--- a/tests/report/test_basic_report.py
+++ b/tests/report/test_basic_report.py
@@ -69,15 +69,13 @@
(0, "Risk Severity Levels"),
(0, "Glossary"),
]
-OFFLINE_OUTLINE = BASE_OUTLINE + [
- (0, "What the Full Assessment Adds"),
- (1, "How to Get Them"),
- (0, "Legal Notices"),
-]
-UPLOADED_OUTLINE = BASE_OUTLINE + [
- (0, "The Full Assessment"),
- (0, "Legal Notices"),
-]
+OUTLINE = BASE_OUTLINE + [(0, "Legal Notices")]
+CLOSING_PAGE_TEXTS = (
+ "What the Full Assessment Adds",
+ "The Full Assessment",
+ "How to Get Them",
+ "HOST",
+)
SCORE_PATTERNS = (
r"/ 100",
r"/ 5\b",
@@ -87,6 +85,14 @@
"Lock-in Score",
"Scored Alternatives",
)
+# An edition name; the "Enterprise support" flag of an alternative is not one
+PROMOTION_PATTERNS = (
+ r"\bLight\b",
+ r"\bEnterprise\b(?!\s+support)",
+ "Exit Readiness Grade",
+ "Vendor Lock-in Score",
+)
+EXITCLOUD_URL = "https://app.exitcloud.io/assessments/a-1"
TFSTATE_SCOPE = {
"file": "infra.tfstate",
"sha256": "ab" * 32,
@@ -252,19 +258,125 @@ def test_a_tfstate_assessment_explains_the_missing_costs(self):
self.assertIn("e1c2f0c0-0000-4000-8000-000000000000 (serial 7)", text)
+class TfstateCoverageTests(BasicReportTestCase):
+ def coverage(self, other=0, unmapped=0):
+ return {
+ "instances_total": 9 + other + unmapped,
+ "instances_counted": 9,
+ "instances_excluded_other_provider": other,
+ "instances_excluded_unmapped": unmapped,
+ }
+
+ def pages(self, key, coverage, **overrides):
+ _path, context, reader = self.render(
+ self.tfstate_fixture(),
+ key=key,
+ tfstate_scope=TFSTATE_SCOPE,
+ tfstate_coverage=coverage,
+ **overrides,
+ )
+ return context, page_texts(reader)
+
+ def test_both_kinds_are_named_with_the_total_in_the_inventory_caption(self):
+ context, pages = self.pages("covered", self.coverage(other=3, unmapped=2))
+
+ self.assertEqual(context.scope.not_assessed, 5)
+ self.assertIn(
+ "Not Assessed 3 resources of other providers, "
+ "2 resources of types not in the data catalogue",
+ pages[1],
+ )
+ inventory = next(text for text in pages if "Share = resources" in text)
+ self.assertIn(
+ "5 resources in the state file were not assessed (see page 2)", inventory
+ )
+
+ def test_only_the_parts_that_are_not_zero_are_shown(self):
+ for key, coverage, expected, absent in (
+ (
+ "other",
+ self.coverage(other=4),
+ "4 resources of other providers",
+ "types not in",
+ ),
+ (
+ "unmapped",
+ self.coverage(unmapped=1),
+ "1 resource of a type not in the data catalogue",
+ "other provider",
+ ),
+ (
+ "one other",
+ self.coverage(other=1),
+ "1 resource of another provider",
+ "types not in",
+ ),
+ ):
+ with self.subTest(key):
+ _context, pages = self.pages(key, coverage)
+ row = pages[1].split("Not Assessed", 1)[1].split("Source", 1)[0]
+
+ self.assertIn(expected, row)
+ self.assertNotIn(absent, row)
+
+ def test_a_singular_total_says_was(self):
+ _context, pages = self.pages("one total", self.coverage(unmapped=1))
+
+ inventory = next(text for text in pages if "Share = resources" in text)
+ self.assertIn(
+ "1 resource in the state file was not assessed (see page 2)", inventory
+ )
+
+ def test_without_excluded_instances_there_is_no_row_and_no_caption(self):
+ for key, coverage in (
+ ("zeros", self.coverage()),
+ ("older manifest", None),
+ ("empty", {}),
+ (
+ "not numbers",
+ {
+ "instances_excluded_other_provider": "3",
+ "instances_excluded_unmapped": -2,
+ },
+ ),
+ ):
+ with self.subTest(key):
+ context, pages = self.pages(key, coverage)
+ text = " ".join(pages)
+
+ self.assertEqual(context.scope.not_assessed, 0)
+ self.assertNotIn("Not Assessed", text)
+ self.assertNotIn("were not assessed", text)
+ self.assertNotIn("was not assessed", text)
+
+ def test_a_live_run_never_shows_the_row(self):
+ _path, context, reader = self.render(
+ key="live coverage", tfstate_coverage=self.coverage(other=3, unmapped=2)
+ )
+
+ self.assertEqual(context.scope.not_assessed, 0)
+ self.assertNotIn("Not Assessed", " ".join(page_texts(reader)))
+
+
class StructureTests(BasicReportTestCase):
def test_the_heading_outline_is_the_expected_list(self):
for name, fixture, overrides in self.scenarios():
with self.subTest(name):
_path, _context, reader = self.render(fixture, key=name, **overrides)
- expected = UPLOADED_OUTLINE if "uploaded" in name else OFFLINE_OUTLINE
- self.assertEqual(outline_entries(reader.outline), expected)
+ self.assertEqual(outline_entries(reader.outline), OUTLINE)
+
+ def test_the_outline_is_the_same_for_an_upload_without_a_link(self):
+ _path, _context, reader = self.render(
+ key="no link", provenance=provenance(uploaded=True, assessment_id="a-77")
+ )
+
+ self.assertEqual(outline_entries(reader.outline), OUTLINE)
def test_the_outline_does_not_depend_on_the_data(self):
_path, _context, reader = self.render(
build_empty_report_fixture(), provenance=provenance()
)
- self.assertEqual(outline_entries(reader.outline), OFFLINE_OUTLINE)
+ self.assertEqual(outline_entries(reader.outline), OUTLINE)
def test_each_chapter_starts_a_page_and_page_two_is_the_document_information(self):
_path, _context, reader = self.render(key="default")
@@ -281,7 +393,9 @@ def test_each_chapter_starts_a_page_and_page_two_is_the_document_information(sel
numbers = [reader.get_destination_page_number(entry) for entry in chapters]
# One chapter per page, except the severity levels and the glossary
self.assertEqual(len(set(numbers)), len(numbers) - 1)
- self.assertEqual(numbers[-4], numbers[-3])
+ self.assertEqual(numbers[-3], numbers[-2])
+ self.assertIn("Legal Notices", pages[numbers[-1]])
+ self.assertEqual(numbers[-1], len(pages) - 1)
def test_running_header_and_footer(self):
path, context, reader = self.render(key="default")
@@ -393,7 +507,7 @@ def test_names_from_the_data_are_escaped(self):
class AlternativesTests(BasicReportTestCase):
- def test_a_missing_vendor_is_left_out_and_alternatives_are_alphabetical(self):
+ def test_a_missing_vendor_shows_a_dash_and_alternatives_are_alphabetical(self):
fixture = build_report_fixture()
technologies = fixture["alternative_technologies"]
technologies.append(
@@ -416,7 +530,11 @@ def test_a_missing_vendor_is_left_out_and_alternatives_are_alphabetical(self):
)
self.assertEqual(entry.alternatives[0].vendor, "")
self.assertEqual(entry.alternatives[1].vendor, "OpenInfra Foundation")
- self.assertNotIn("Unknown Organization", " ".join(page_texts(reader)))
+ text = " ".join(page_texts(reader))
+ self.assertIn("Alpha · –", text)
+ self.assertIn("OpenStack · OpenInfra Foundation", text)
+ self.assertLess(text.index("Alpha · –"), text.index("OpenStack ·"))
+ self.assertNotIn("Unknown Organization", text)
class ProvenanceTests(BasicReportTestCase):
@@ -433,33 +551,71 @@ def test_an_offline_report_says_it_is_self_collected(self):
self.assertIn(SYNC_ID, second)
self.assertIn("2026-11-02 · (sha256 cccccccccccc)", second)
self.assertIn("Self-collected data: no platform evaluated this report.", second)
- self.assertIn("What the Full Assessment Adds", pages[-2])
self.assertEqual(
link_targets(reader) - AWS_NOTICE_LINKS,
- {"https://escapecloud.io", "https://exitcloud.io"},
+ set(),
)
- def test_an_uploaded_report_carries_the_link_instead_of_the_promotion(self):
+ def test_an_uploaded_report_carries_the_link_on_page_two_only(self):
_path, _context, reader = self.render(key="uploaded", provenance=uploaded())
pages = page_texts(reader)
self.assertIn("Uploaded to the platform", pages[1])
self.assertIn(URL, pages[1])
self.assertNotIn("Self-collected data", " ".join(pages))
- self.assertIn("The Full Assessment", pages[-2])
- self.assertIn(URL, pages[-2])
- self.assertNotIn("What the Full Assessment Adds", " ".join(pages))
+ self.assertEqual(sum(URL in text for text in pages), 1)
self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, {URL})
- def test_an_upload_without_a_link_names_the_assessment(self):
+ def test_an_upload_without_a_link_names_the_assessment_on_page_two(self):
_path, _context, reader = self.render(
- provenance=(
- uploaded(platform_url=None)
- if False
- else provenance(uploaded=True, platform_url=None, assessment_id="a-77")
- )
+ key="no link", provenance=provenance(uploaded=True, assessment_id="a-77")
+ )
+ pages = page_texts(reader)
+
+ self.assertIn("Uploaded to the platform", pages[1])
+ self.assertIn("Assessment ID: a-77", pages[1])
+ self.assertEqual(sum("a-77" in text for text in pages), 1)
+ self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, set())
+
+ def test_a_failed_upload_says_what_happened(self):
+ for name, changes, expected in (
+ (
+ "kept",
+ {"upload_reason": "upload_unavailable", "upload_spooled": True},
+ "Upload failed: upload_unavailable. The run was kept for a later upload.",
+ ),
+ (
+ "lost",
+ {"upload_reason": "upload_rejected"},
+ "Upload failed: upload_rejected.",
+ ),
+ ("no reason", {}, "Upload failed."),
+ (
+ "no reason, kept",
+ {"upload_spooled": True},
+ "Upload failed. The run was kept",
+ ),
+ ):
+ with self.subTest(name):
+ _path, _context, reader = self.render(
+ key=f"failed {name}",
+ provenance=provenance(upload_failed=True, **changes),
+ )
+ pages = page_texts(reader)
+
+ self.assertIn(expected, pages[1])
+ self.assertNotIn("Self-collected data", " ".join(pages))
+ self.assertEqual("kept for a later upload" in pages[1], "kept" in name)
+ self.assertEqual(link_targets(reader) - AWS_NOTICE_LINKS, set())
+
+ def test_a_reason_that_ends_in_a_full_stop_is_not_doubled(self):
+ _path, _context, reader = self.render(
+ key="failed stop",
+ provenance=provenance(upload_failed=True, upload_reason="No answer."),
)
- self.assertIn("Assessment ID: a-77", page_texts(reader)[-2])
+
+ self.assertIn("Upload failed: No answer. ", page_texts(reader)[1] + " ")
+ self.assertNotIn("No answer..", page_texts(reader)[1])
def test_an_unknown_catalogue_version_is_said_so(self):
_path, _context, reader = self.render(
@@ -554,31 +710,71 @@ def test_the_json_report_has_a_provenance_object(self):
class BasicOnlyTests(BasicReportTestCase):
- def test_no_score_or_grade_appears_outside_the_closing_page(self):
+ def test_no_score_or_grade_appears_on_any_page(self):
for name, fixture, overrides in self.scenarios():
with self.subTest(name):
_path, _context, reader = self.render(fixture, key=name, **overrides)
- body = " ".join(page_texts(reader)[:-2])
+ body = " ".join(page_texts(reader))
for pattern in SCORE_PATTERNS:
self.assertIsNone(re.search(pattern, body, re.IGNORECASE), pattern)
- def test_the_closing_page_only_names_what_the_full_assessment_adds(self):
- _path, _context, reader = self.render(key="offline", provenance=provenance())
- closing = page_texts(reader)[-2]
+ def test_no_closing_page_and_no_command_is_left(self):
+ for name, fixture, overrides in self.scenarios():
+ with self.subTest(name):
+ _path, _context, reader = self.render(fixture, key=name, **overrides)
+ body = " ".join(page_texts(reader))
- for name in (
- "Exit Readiness Grade",
- "Vendor Lock-in Score",
- "Scored Alternatives",
- ):
- self.assertIn(name, closing)
- # The Community Edition produces it itself, so the platforms do not add it
- self.assertNotIn("Egress & Data Landscape", closing)
- self.assertFalse(
- any("Egress" in term for term, _text in BLOCKS["fa_items"]),
- )
- self.assertNotRegex(closing, r"\d+ / (100|5)")
+ for text in CLOSING_PAGE_TEXTS:
+ self.assertNotIn(text, body)
+ self.assertNotIn("--api-key-file", body)
+
+ def test_no_edition_or_grade_is_promoted_on_any_page(self):
+ for name, fixture, overrides in self.scenarios():
+ with self.subTest(name):
+ _path, _context, reader = self.render(fixture, key=name, **overrides)
+ for number, text in enumerate(page_texts(reader), start=1):
+ for pattern in PROMOTION_PATTERNS:
+ self.assertIsNone(re.search(pattern, text), (number, pattern))
+
+ def test_the_enterprise_support_flag_does_not_trip_the_edition_check(self):
+ _path, _context, reader = self.render(key="default")
+ body = " ".join(page_texts(reader))
+
+ self.assertIn("Enterprise support", body)
+ self.assertIsNone(re.search(PROMOTION_PATTERNS[1], body))
+ self.assertIsNotNone(re.search(PROMOTION_PATTERNS[1], "The Enterprise edition"))
+ self.assertIsNotNone(re.search(PROMOTION_PATTERNS[0], "A Light assessment"))
+
+ def test_exitcloud_is_named_only_on_page_two_of_an_uploaded_report(self):
+ for name, fixture, overrides in self.scenarios():
+ if "uploaded" in name:
+ overrides = {
+ **overrides,
+ "provenance": provenance(
+ uploaded=True, platform_url=EXITCLOUD_URL, assessment_id="a-1"
+ ),
+ }
+ with self.subTest(name):
+ _path, _context, reader = self.render(fixture, **overrides)
+ pages = [text.lower() for text in page_texts(reader)]
+ named = [
+ n for n, text in enumerate(pages, start=1) if "exitcloud" in text
+ ]
+
+ self.assertEqual(named, [2] if "uploaded" in name else [])
+
+ def test_escapecloud_is_named_only_on_the_legal_notices_page(self):
+ for name, fixture, overrides in self.scenarios():
+ with self.subTest(name):
+ _path, _context, reader = self.render(fixture, key=name, **overrides)
+ pages = [text.lower() for text in page_texts(reader)]
+ named = [
+ n for n, text in enumerate(pages, start=1) if "escapecloud" in text
+ ]
+
+ self.assertEqual(named, [len(pages)])
+ self.assertIn("legal notices", pages[-1])
class TermGuardTests(BasicReportTestCase):
@@ -623,19 +819,6 @@ def test_the_guard_finds_what_it_should(self):
self.assertEqual(find_terms("an ordinary sentence about cloud exit"), [])
self.assertEqual(find_terms("ICTs and DORAs"), [])
- def test_the_platforms_are_named_only_on_the_last_two_pages(self):
- _path, _context, reader = self.render(key="default")
- pages = page_texts(reader)
-
- for text in pages[:-2]:
- self.assertNotIn("escapecloud", text.lower())
- self.assertNotIn("exitcloud", text.lower())
- # The closing page keeps both platform URLs; the notices name EscapeCloud
- self.assertIn("https://escapecloud.io", pages[-2])
- self.assertIn("https://exitcloud.io", pages[-2])
- self.assertIn("EscapeCloud", pages[-1])
- self.assertNotIn("exitcloud", pages[-1].lower())
-
class LegalNoticesTests(BasicReportTestCase):
ICON_MARKERS = {1: "Microsoft group", 2: "Amazon.com"}
@@ -685,6 +868,21 @@ def test_the_icon_links_follow_the_provider(self):
self.assertIn(AZURE_ICONS_URL, links)
self.assertNotIn(AWS_ICONS_URL, links)
+ def test_the_icon_names_are_followed_by_their_addresses(self):
+ expected = {
+ 2: [
+ f"AWS Architecture Icons ({AWS_ICONS_URL})",
+ f"AWS Trademark Guidelines ({AWS_TRADEMARK_URL})",
+ ],
+ 1: [f"Azure Architecture Icons ({AZURE_ICONS_URL})"],
+ }
+ for name, fixture, overrides in self.scenarios():
+ with self.subTest(name):
+ context, _reader, pages = self.notices_page(name, fixture, **overrides)
+ last = pages[-1].replace(" ", "")
+ for text in expected[context.cloud_service_provider]:
+ self.assertIn(text.replace(" ", ""), last)
+
def test_the_cover_names_the_edition_and_no_classification(self):
_context, _reader, pages = self.notices_page("offline", provenance=provenance())
cover = pages[0]
diff --git a/tests/report/test_html_report.py b/tests/report/test_html_report.py
index 128569d..de594af 100644
--- a/tests/report/test_html_report.py
+++ b/tests/report/test_html_report.py
@@ -11,7 +11,12 @@
from pypdf import PdfReader
from cloudexit.engine import generate_report, perform_risk_assessment
-from cloudexit.report.sections import AWS_ICONS_URL, AZURE_ICONS_URL, REPOSITORY_URL
+from cloudexit.report.sections import (
+ AWS_ICONS_URL,
+ AWS_TRADEMARK_URL,
+ AZURE_ICONS_URL,
+ REPOSITORY_URL,
+)
from cloudexit.report.wording import BLOCKS
from cloudexit.resources import asset_path
from tests.report.test_basic_report import page_texts
@@ -227,9 +232,8 @@ def test_the_uploaded_chip_and_link_appear_only_on_uploaded_runs(self):
chip = re.search(r']*>{name}', page
+ )
+ ),
+ 1,
+ name,
+ )
+ self.assertNotIn(f"({name})", text)
+ self.assertNotIn(f"( {name} )", text)
+ self.assertNotIn(f"{name} (", text)
+ for other in names.keys() - {scenario.provider}:
+ for name in names[other]:
+ self.assertNotIn(name, text)
+
def test_the_figures_notice_follows_the_state(self):
def figures(case):
page = self.page(case)
diff --git a/tests/report/test_report_flow.py b/tests/report/test_report_flow.py
index 1990737..6277a8c 100644
--- a/tests/report/test_report_flow.py
+++ b/tests/report/test_report_flow.py
@@ -5,6 +5,7 @@
import sqlite3
import tempfile
import unittest
+from dataclasses import replace
from contextlib import nullcontext
from pathlib import Path
from unittest.mock import patch
@@ -23,6 +24,7 @@
upload,
write_upload_record,
)
+from tests.support.html import html_text
from tests.support.isolation import forget_cache, isolated_environ
from tests.support.api import (
ASSESSMENT_ID,
@@ -184,7 +186,7 @@ def test_an_offline_report_has_no_upload_trace(self):
sorted(int(entry["id"]) for entry in document["data"]["risk_inventory"]),
[2, 4],
)
- self.assertNotIn("Not uploaded", self.html(report))
+ self.assertNotIn("Upload failed", self.html(report))
self.assertTrue(Path(result["reports"]["PDF"]).is_file())
def generate(self, report, upload_info):
@@ -207,10 +209,29 @@ def test_a_failed_upload_puts_a_banner_in_every_report(self):
result = self.generate(report, info)
self.assertTrue(result["success"], result.get("logs"))
- self.assertIn("Not uploaded: upload_unavailable", self.html(report))
+ self.assertIn("Upload failed: upload_unavailable.", self.html(report))
self.assertEqual(self.json_report(report)["meta"]["upload"], info)
self.assertTrue(Path(result["reports"]["PDF"]).is_file())
+ def test_a_failed_upload_that_was_kept_says_so_in_the_provenance(self):
+ report = self.report()
+ perform_risk_assessment(1, report)
+ info = {
+ "status": "failed",
+ "reason": REASON_UNAVAILABLE,
+ "sync_id": SYNC_ID,
+ "spooled": True,
+ }
+
+ result = self.generate(report, info)
+
+ self.assertTrue(result["success"], result.get("logs"))
+ self.assertIn(
+ "Upload failed: upload_unavailable. The run was kept for a later upload.",
+ html_text(self.html(report)),
+ )
+ self.assertEqual(self.json_report(report)["meta"]["upload"], info)
+
def test_a_successful_upload_records_only_the_link(self):
report = self.report()
perform_risk_assessment(1, report)
@@ -347,6 +368,17 @@ def test_report_info_and_the_record_file(self):
self.assertIsNone(report_info(None))
self.assertIsNone(report_info(UploadResult(False, False)))
+ kept = replace(failed, spooled=True)
+ self.assertEqual(
+ report_info(kept),
+ {
+ "status": "failed",
+ "reason": REASON_REJECTED,
+ "sync_id": SYNC_ID,
+ "spooled": True,
+ },
+ )
+
detailed = UploadResult(
True,
True,
@@ -547,7 +579,7 @@ def test_a_failed_upload_still_writes_the_report_and_exits_with_an_upload_code(
self.assertEqual(ctx.exception.code, exit_code)
self.assert_local_report(report)
- self.assertIn(f"Not uploaded: {reason}", self.html(report))
+ self.assertIn(f"Upload failed: {reason}.", self.html(report))
self.assertEqual(
self.json_report(report)["meta"]["upload"]["reason"], reason
)
@@ -650,7 +682,7 @@ def test_a_key_that_cannot_be_right_is_a_failed_upload_not_an_offline_run(self):
mock_upload.assert_not_called()
self.assertEqual(ctx.exception.code, codes.UPLOAD_AUTH)
self.assert_local_report(report)
- self.assertIn("Not uploaded: upload_auth", self.html(report))
+ self.assertIn("Upload failed: upload_auth.", self.html(report))
# A configured platform keeps the requested depth: nothing forces Basic
self.assertEqual(self.sync(report)["assessment_type"], 2)
messages = [args[0] for args, _ in self.steps]
@@ -691,7 +723,7 @@ def test_collection_problems_win_over_upload_codes(self):
self.assertEqual(ctx.exception.code, codes.EGRESS)
self.assert_local_report(report)
- self.assertIn("Not uploaded", self.html(report))
+ self.assertIn("Upload failed", self.html(report))
def test_a_sync_document_problem_wins_over_an_upload_code(self):
report = self.report()
diff --git a/tests/samples/render.py b/tests/samples/render.py
index de5d4b5..84eaaaf 100644
--- a/tests/samples/render.py
+++ b/tests/samples/render.py
@@ -1,6 +1,7 @@
import logging
import shutil
import sqlite3
+import tempfile
from contextlib import ExitStack
from dataclasses import dataclass, field
from datetime import datetime
@@ -228,6 +229,19 @@ def _render(scenario, directory, catalogue_path):
raise RenderError(f"egress.pdf failed: {report['egress_pdf_logs']}")
+def _legacy_copy(catalogue_path, directory):
+ # The dataset of an older release: the same tables, no catalogue_meta
+ copy = Path(directory) / "legacy_catalogue.db"
+ shutil.copy(catalogue_path, copy)
+ conn = sqlite3.connect(copy)
+ try:
+ conn.execute("DROP TABLE IF EXISTS catalogue_meta")
+ conn.commit()
+ finally:
+ conn.close()
+ return copy
+
+
def render_case(scenario, out_dir, *, catalogue_path):
directory = Path(out_dir) / scenario.case
if directory.exists():
@@ -237,8 +251,11 @@ def render_case(scenario, out_dir, *, catalogue_path):
previous = store._active_override
clock = frozen_datetime(scenario.clock)
try:
- store.set_active_catalogue(catalogue_path)
with ExitStack() as stack:
+ if scenario.catalogue == "legacy":
+ scratch = stack.enter_context(tempfile.TemporaryDirectory())
+ catalogue_path = _legacy_copy(catalogue_path, scratch)
+ store.set_active_catalogue(catalogue_path)
# The warnings of the pipeline are about the sample data, not news
stack.callback(logging.disable, logging.root.manager.disable)
logging.disable(logging.WARNING)
diff --git a/tests/samples/scenarios.py b/tests/samples/scenarios.py
index 94a2f46..a531f25 100644
--- a/tests/samples/scenarios.py
+++ b/tests/samples/scenarios.py
@@ -23,11 +23,14 @@
"clock",
"run_id",
)
+OPTIONAL_FIELDS = ("catalogue",)
+CATALOGUES = ("legacy",)
SCOPE_FIELDS = {
"aws": ("accountId", "region"),
"azure": ("tenantId", "subscriptionId", "resourceGroupName"),
}
-UPLOAD_FIELDS = ("status", "url", "assessment_id", "sync_id", "reason")
+UPLOAD_FIELDS = ("status", "url", "assessment_id", "sync_id", "reason", "spooled")
+UPLOAD_STATUSES = ("uploaded", "failed")
MONTH = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
CURRENCY = re.compile(r"^[A-Z]{3}$")
UUID7 = re.compile(
@@ -54,6 +57,8 @@ class Scenario:
upload: dict | None
clock: datetime
run_id: str
+ # "legacy": a catalogue without catalogue_meta
+ catalogue: str | None = None
@property
def csp(self):
@@ -218,10 +223,22 @@ def _upload(path, upload):
if not isinstance(upload, dict):
_fail(path, "upload", "expected an object or null")
_only(path, "upload", upload, UPLOAD_FIELDS)
- _text(path, "upload.status", upload.get("status"))
+ status = _text(path, "upload.status", upload.get("status"))
+ if status not in UPLOAD_STATUSES:
+ _fail(path, "upload.status", f"expected one of {list(UPLOAD_STATUSES)}")
+ if "reason" in upload:
+ _text(path, "upload.reason", upload["reason"])
+ if "spooled" in upload and not isinstance(upload["spooled"], bool):
+ _fail(path, "upload.spooled", "expected true or false")
return upload
+def _catalogue(path, value):
+ if value is not None and value not in CATALOGUES:
+ _fail(path, "catalogue", f"expected one of {list(CATALOGUES)}, got {value!r}")
+ return value
+
+
def _clock(path, value):
text = _text(path, "clock", value)
if not text.endswith("Z"):
@@ -243,7 +260,7 @@ def load_scenario(path):
for field in FIELDS:
if field not in data:
_fail(path, field, "missing")
- _only(path, "(file)", data, FIELDS)
+ _only(path, "(file)", data, FIELDS + OPTIONAL_FIELDS)
provider = data["provider"]
if provider not in PROVIDER_IDS:
@@ -280,6 +297,7 @@ def load_scenario(path):
upload=_upload(path, data["upload"]),
clock=_clock(path, data["clock"]),
run_id=run_id,
+ catalogue=_catalogue(path, data.get("catalogue")),
)
diff --git a/tests/samples/test_samples.py b/tests/samples/test_samples.py
index 8026ed3..26524b6 100644
--- a/tests/samples/test_samples.py
+++ b/tests/samples/test_samples.py
@@ -224,35 +224,54 @@ def test_the_egress_notices_carry_the_egress_paragraph(self):
class UploadTests(RenderedTestCase):
- def test_uploaded_cases_show_their_link_or_assessment_id(self):
+ def test_uploaded_cases_show_their_link_or_assessment_id_on_page_two_only(self):
linked = STATE["scenarios"]["azure_uploaded"].upload["url"]
for pdf in ("report.pdf", "egress.pdf"):
pages = self.pdf_pages("azure_uploaded", pdf)
self.assertIn(linked, squashed(pages[1]))
self.assertIn("Uploaded to the platform", pages[1])
- self.assertIn(
- "The Full Assessment", self.pdf_pages("azure_uploaded", "report.pdf")[-2]
- )
- self.assertIn(
- linked, squashed(self.pdf_pages("azure_uploaded", "egress.pdf")[-2])
+ if pdf == "report.pdf":
+ self.assertEqual(sum(linked in squashed(text) for text in pages), 1)
+ self.assertNotIn(
+ "The Full Assessment",
+ " ".join(self.pdf_pages("azure_uploaded", "report.pdf")),
)
pages = self.pdf_pages("uploaded_no_link", "report.pdf")
self.assertIn("Uploaded to the platform", pages[1])
- self.assertIn("Assessment ID: sample-assessment-0001", pages[-2])
+ self.assertIn("Assessment ID: sample-assessment-0001", pages[1])
+ self.assertEqual(sum("sample-assessment-0001" in text for text in pages), 1)
self.assertNotIn("https://platform", " ".join(pages))
- def test_offline_cases_show_the_self_collected_line(self):
+ def test_offline_cases_show_the_self_collected_line_and_no_closing_page(self):
for case, scenario in self.cases():
if scenario.upload is None:
with self.subTest(case):
pages = self.pdf_pages(case, "report.pdf")
self.assertIn("Self-collected data", pages[1])
- self.assertIn("What the Full Assessment Adds", " ".join(pages))
+ self.assertNotIn("What the Full Assessment Adds", " ".join(pages))
+ self.assertIn("Legal Notices", pages[-1][:200])
if scenario.egress is not None:
egress = self.pdf_pages(case, "egress.pdf")
self.assertIn("Self-collected data", egress[1])
+ def test_a_failed_upload_says_what_happened_in_both_pdfs_and_the_html(self):
+ wording = (
+ "Upload failed: The platform did not answer in time. "
+ "The run was kept for a later upload."
+ )
+ for pdf in ("report.pdf", "egress.pdf"):
+ if (self.folder("upload_failed") / pdf).is_file():
+ self.assertIn(wording, self.pdf_pages("upload_failed", pdf)[1])
+ pages = self.pdf_pages("upload_failed", "report.pdf")
+ self.assertNotIn("Self-collected data", " ".join(pages))
+ self.assertIn(wording, html_text(self.text("upload_failed", "index.html")))
+ chip = re.search(
+ r'