diff --git a/.claude/commands/cypress/cypress-run.md b/.claude/commands/cypress/cypress-run.md index e5f9b6caf..3b50e9962 100644 --- a/.claude/commands/cypress/cypress-run.md +++ b/.claude/commands/cypress/cypress-run.md @@ -341,23 +341,27 @@ npm run cypress:run -- --headed --config video=false Use these tags with `--env grepTags`: -**Feature Tags:** +**1. Modifier Tags:** + +- `@flaky` - Tests that don't pass reliably +- `@xfail` - Tests for known bugs expected to fail +- `@slow` - Long-running e2e tests (15+ minutes) + +**2. Infrastructure Tags:** + +- `@acm` - Advanced Cluster Management integration tests +- `@coo` - Cluster Observability Operator functionality tests +- `@virtualization` - Virtualization integration tests +- `@ols` - OpenShift Lightspeed. Requires external installation of OLS and setup of LLM accounts through CI configuration + +**3. Feature Tags** (format: `@{component}`): -- `@acm-alerting` - Alert-related tests in ACM perspective - `@alerting` - Alert-related tests - `@legacy-dashboards` - Legacy dashboard tests - `@metrics` - Metrics explorer tests +- `@targets` - Targets tests - `@perses-dashboards` - Perses dashboard tests -- `@coo` - Observability Operator tests -- `@ols` - OpenShift Lightspeed integration tests -- `@virtualization` - OpenShift Virtualization tests - `@cluster-health-analyzer` - Incidents feature tests -- `@targets` - Targets page tests - -**Modifier Tags:** - -- `@slow` - Longer running tests -- `@flaky` - Known flaky tests --- diff --git a/web/cypress.config.ts b/web/cypress.config.ts index 492d3cf81..dad9bec9b 100644 --- a/web/cypress.config.ts +++ b/web/cypress.config.ts @@ -5,8 +5,8 @@ import * as path from 'path'; import registerCypressGrep from '@cypress/grep/src/plugin'; import { DefinePlugin, NormalModuleReplacementPlugin } from 'webpack'; import { - writeBenchmarkReport, injectBenchmarksIntoMochawesome, + writeBenchmarkReport, } from './cypress/plugins/benchmark-reporter'; const getLoginCredentials = (index: number): { username: string; password: string } => { diff --git a/web/cypress/README.md b/web/cypress/README.md index 10db1be08..4c53a6941 100644 --- a/web/cypress/README.md +++ b/web/cypress/README.md @@ -214,21 +214,21 @@ Tests are organized using tags for selective execution using [@cypress/grep](htt #### Tag Categories -**1. Basic Tags:** +**1. Modifier Tags:** - `@flaky` - Tests that don't pass reliably - `@xfail` - Tests for known bugs expected to fail - `@slow` - Long-running e2e tests (15+ minutes) -**2. High-Level Component Tags:** +**2. Infrastructure Tags:** -- `@coo` - Cluster Observability Operator functionality tests (operator installation, ACM integration) +- `@acm` - Advanced Cluster Management integration tests +- `@coo` - Cluster Observability Operator functionality tests - `@virtualization` - Virtualization integration tests - `@ols` - OpenShift Lightspeed. Requires external installation of OLS and setup of LLM accounts through CI configuration -**3. Specific Feature Tags** (format: `@{component}`): +**3. Feature Tags** (format: `@{component}`): -- `@acm-alerting` - Alert-related tests in ACM perspective - `@alerting` - Alert-related tests - `@legacy-dashboards` - Legacy dashboard tests - `@metrics` - Metrics explorer tests @@ -236,11 +236,6 @@ Tests are organized using tags for selective execution using [@cypress/grep](htt - `@perses-dashboards` - Perses dashboard tests - `@cluster-health-analyzer` - Incidents feature tests -**4. JIRA Tags** (format: `@JIRA-{ID}`): - -- Example: `@JIRA-OU-1033` -- Link tests to specific JIRA issues - #### Running Tests by Tags **Run component-specific tests:** diff --git a/web/cypress/e2e/coo/01.coo_bvt.cy.ts b/web/cypress/e2e/coo/01.coo_bvt.cy.ts index eba132f12..0454d0815 100644 --- a/web/cypress/e2e/coo/01.coo_bvt.cy.ts +++ b/web/cypress/e2e/coo/01.coo_bvt.cy.ts @@ -18,7 +18,7 @@ const MP = { operatorName: 'Cluster Monitoring Operator', }; -describe('BVT: COO', { tags: ['@alerting', '@acm-alerting', '@coo'] }, () => { +describe('BVT: COO', { tags: ['@alerting', '@coo'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); }); diff --git a/web/cypress/e2e/coo/02.acm_alerting_ui.cy.ts b/web/cypress/e2e/coo/02.acm_alerting_ui.cy.ts index fd200d879..b85594707 100644 --- a/web/cypress/e2e/coo/02.acm_alerting_ui.cy.ts +++ b/web/cypress/e2e/coo/02.acm_alerting_ui.cy.ts @@ -24,7 +24,7 @@ const MP = { }; const expectedAlerts = ['Watchdog', 'Watchdog-spoke', 'ClusterCPUHealth-jb']; -describe('ACM Alerting UI', { tags: ['@alerting', '@acm-alerting', '@coo'] }, () => { +describe('ACM Alerting UI', { tags: ['@alerting', '@acm', '@coo'] }, () => { before(() => { cy.beforeBlockACM(MCP, MP); }); diff --git a/web/cypress/e2e/coo/02.acm_perses.cy.ts b/web/cypress/e2e/coo/02.acm_perses.cy.ts index e0c82483b..e32fe2185 100644 --- a/web/cypress/e2e/coo/02.acm_perses.cy.ts +++ b/web/cypress/e2e/coo/02.acm_perses.cy.ts @@ -18,7 +18,7 @@ const MP = { operatorName: 'Cluster Monitoring Operator', }; -describe('ACM - Perses', { tags: ['@coo', '@perses-dashboards'] }, () => { +describe('ACM - Perses', { tags: ['@perses-dashboards', '@acm', '@coo'] }, () => { before(() => { cy.beforeBlockACM(MCP, MP); cy.switchPerspective('Core platform'); diff --git a/web/cypress/e2e/incidents/00.coo_incidents_e2e.cy.ts b/web/cypress/e2e/incidents/00.coo_incidents_e2e.cy.ts index f6cf6664d..fd0f6a75f 100644 --- a/web/cypress/e2e/incidents/00.coo_incidents_e2e.cy.ts +++ b/web/cypress/e2e/incidents/00.coo_incidents_e2e.cy.ts @@ -20,7 +20,7 @@ const MP = { operatorName: 'Cluster Monitoring Operator', }; -describe('BVT: Incidents - e2e', { tags: ['@slow', '@cluster-health-analyzer', '@coo'] }, () => { +describe('BVT: Incidents - e2e', { tags: ['@cluster-health-analyzer', '@coo', '@slow'] }, () => { let currentAlertName: string; before(() => { diff --git a/web/cypress/e2e/incidents/regression/02.reg_ui_tooltip_boundary_times.cy.ts b/web/cypress/e2e/incidents/regression/02.reg_ui_tooltip_boundary_times.cy.ts index e14001afb..41d94117d 100644 --- a/web/cypress/e2e/incidents/regression/02.reg_ui_tooltip_boundary_times.cy.ts +++ b/web/cypress/e2e/incidents/regression/02.reg_ui_tooltip_boundary_times.cy.ts @@ -30,7 +30,7 @@ const MP = { describe( 'Regression: Mixed Severity Interval Boundary Times', - { tags: ['@cluster-health-analyzer', '@xfail', '@coo'] }, + { tags: ['@cluster-health-analyzer', '@coo', '@xfail'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP, { dashboards: false, troubleshootingPanel: false }); diff --git a/web/cypress/e2e/incidents/regression/03-04.reg_e2e_firing_alerts.cy.ts b/web/cypress/e2e/incidents/regression/03-04.reg_e2e_firing_alerts.cy.ts index 3496ec6ca..f1eb8ec51 100644 --- a/web/cypress/e2e/incidents/regression/03-04.reg_e2e_firing_alerts.cy.ts +++ b/web/cypress/e2e/incidents/regression/03-04.reg_e2e_firing_alerts.cy.ts @@ -36,7 +36,7 @@ const MP = { describe( 'Regression: Time-Based Alert Resolution (E2E with Firing Alerts)', - { tags: ['@cluster-health-analyzer', '@slow', '@coo'] }, + { tags: ['@cluster-health-analyzer', '@coo', '@slow'] }, () => { let currentAlertName: string; diff --git a/web/cypress/e2e/perses/05.coo_tempo_loki_admin.cy.ts b/web/cypress/e2e/perses/05.coo_tempo_loki_admin.cy.ts index 846657280..1bff57b53 100644 --- a/web/cypress/e2e/perses/05.coo_tempo_loki_admin.cy.ts +++ b/web/cypress/e2e/perses/05.coo_tempo_loki_admin.cy.ts @@ -43,7 +43,7 @@ const CLO = { describe( 'COO - Dashboards (Perses) - Perses Global Datasources with Tempo and Loki', - { tags: ['@perses-dashboards', '@xfail', '@coo'] }, + { tags: ['@perses-dashboards', '@coo', '@xfail'] }, () => { before(() => { cy.beforeBlockTempo(TEMPO); diff --git a/web/cypress/e2e/virtualization/00.coo_ivt.cy.ts b/web/cypress/e2e/virtualization/00.coo_ivt.cy.ts index 8a379395e..eeda8cbea 100644 --- a/web/cypress/e2e/virtualization/00.coo_ivt.cy.ts +++ b/web/cypress/e2e/virtualization/00.coo_ivt.cy.ts @@ -34,8 +34,8 @@ const KBV = { }; describe( - 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@virtualization', '@coo', '@slow'] }, + 'IVT: Monitoring + Virtualization', + { tags: ['@alerting', '@metrics', '@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -47,7 +47,7 @@ describe( }, ); -describe('Installation: Virtualization', { tags: ['@virtualization', '@coo', '@slow'] }, () => { +describe('Installation: Virtualization', { tags: ['@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockVirtualization(KBV); }); @@ -61,7 +61,7 @@ describe('Installation: Virtualization', { tags: ['@virtualization', '@coo', '@s describe( 'IVT: Monitoring + Virtualization', - { tags: ['@metrics', '@alerting', '@virtualization', '@coo'] }, + { tags: ['@alerting', '@metrics', '@coo', '@virtualization'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/e2e/virtualization/01.coo_ivt_alerts.cy.ts b/web/cypress/e2e/virtualization/01.coo_ivt_alerts.cy.ts index 090700965..581bf7060 100644 --- a/web/cypress/e2e/virtualization/01.coo_ivt_alerts.cy.ts +++ b/web/cypress/e2e/virtualization/01.coo_ivt_alerts.cy.ts @@ -34,8 +34,8 @@ const KBV = { }; describe( - 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@alerting', '@virtualization', '@slow', '@coo'] }, + 'Regression: Monitoring - Alerts (Virtualization)', + { tags: ['@alerting', '@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -49,7 +49,7 @@ describe( describe( 'IVT: Monitoring UIPlugin + Virtualization', - { tags: ['@alerting', '@virtualization', '@slow', '@coo'] }, + { tags: ['@alerting', '@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockVirtualization(KBV); @@ -65,7 +65,7 @@ describe( describe( 'Regression: Monitoring - Alerts (Virtualization)', - { tags: ['@alerting', '@slow', '@virtualization', '@coo'] }, + { tags: ['@alerting', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/e2e/virtualization/02.coo_ivt_metrics_1.cy.ts b/web/cypress/e2e/virtualization/02.coo_ivt_metrics_1.cy.ts index 9c84c795c..432a989d2 100644 --- a/web/cypress/e2e/virtualization/02.coo_ivt_metrics_1.cy.ts +++ b/web/cypress/e2e/virtualization/02.coo_ivt_metrics_1.cy.ts @@ -34,8 +34,8 @@ const KBV = { }; describe( - 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@virtualization', '@slow', '@coo'] }, + 'Regression: Monitoring - Metrics (Virtualization)', + { tags: ['@metrics', '@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -49,7 +49,7 @@ describe( describe( 'IVT: Monitoring UIPlugin + Virtualization', - { tags: ['@virtualization', '@slow', '@coo'] }, + { tags: ['@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockVirtualization(KBV); @@ -65,7 +65,7 @@ describe( describe( 'Regression: Monitoring - Metrics (Virtualization)', - { tags: ['@metrics', '@slow', '@virtualization', '@coo'] }, + { tags: ['@metrics', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); @@ -87,7 +87,7 @@ describe( describe( 'Regression: Monitoring - Metrics Namespaced (Virtualization)', - { tags: ['@metrics', '@slow', '@virtualization', '@coo'] }, + { tags: ['@metrics', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/e2e/virtualization/02.coo_ivt_metrics_2.cy.ts b/web/cypress/e2e/virtualization/02.coo_ivt_metrics_2.cy.ts index 46ebf954a..c48221c4d 100644 --- a/web/cypress/e2e/virtualization/02.coo_ivt_metrics_2.cy.ts +++ b/web/cypress/e2e/virtualization/02.coo_ivt_metrics_2.cy.ts @@ -35,7 +35,7 @@ const KBV = { describe( 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@virtualization', '@slow', '@coo'] }, + { tags: ['@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -49,7 +49,7 @@ describe( describe( 'IVT: Monitoring UIPlugin + Virtualization', - { tags: ['@virtualization', '@slow', '@coo'] }, + { tags: ['@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockVirtualization(KBV); @@ -65,7 +65,7 @@ describe( describe( 'Regression: Monitoring - Metrics (Virtualization)', - { tags: ['@metrics', '@slow', '@virtualization', '@coo'] }, + { tags: ['@metrics', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); @@ -87,7 +87,7 @@ describe( describe( 'Regression: Monitoring - Metrics Namespaced (Virtualization)', - { tags: ['@metrics', '@slow', '@virtualization', '@coo'] }, + { tags: ['@metrics', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/e2e/virtualization/03.coo_ivt_legacy_dashboards.cy.ts b/web/cypress/e2e/virtualization/03.coo_ivt_legacy_dashboards.cy.ts index 97dcb3705..6a5f38d37 100644 --- a/web/cypress/e2e/virtualization/03.coo_ivt_legacy_dashboards.cy.ts +++ b/web/cypress/e2e/virtualization/03.coo_ivt_legacy_dashboards.cy.ts @@ -33,8 +33,8 @@ const KBV = { }; describe( - 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@virtualization', '@slow', '@coo'] }, + 'Regression: Monitoring - Legacy Dashboards (Virtualization)', + { tags: ['@legacy-dashboards', '@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -48,7 +48,7 @@ describe( describe( 'IVT: Monitoring UIPlugin + Virtualization', - { tags: ['@virtualization', '@slow', '@coo'] }, + { tags: ['@coo', '@virtualization', '@slow'] }, () => { before(() => { cy.beforeBlockVirtualization(KBV); @@ -64,7 +64,7 @@ describe( describe( 'Regression: Monitoring - Legacy Dashboards (Virtualization)', - { tags: ['@legacy-dashboards', '@slow', '@virtualization', '@coo'] }, + { tags: ['@legacy-dashboards', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); @@ -84,7 +84,7 @@ describe( describe( 'Regression: Monitoring - Legacy Dashboards Namespaced (Virtualization)', - { tags: ['@legacy-dashboards', '@slow', '@virtualization', '@coo'] }, + { tags: ['@legacy-dashboards', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/e2e/virtualization/04.coo_ivt_perses.cy.ts b/web/cypress/e2e/virtualization/04.coo_ivt_perses.cy.ts index 92654ba0a..cc4d61723 100644 --- a/web/cypress/e2e/virtualization/04.coo_ivt_perses.cy.ts +++ b/web/cypress/e2e/virtualization/04.coo_ivt_perses.cy.ts @@ -33,8 +33,9 @@ const KBV = { }; describe( - 'Installation: COO and setting up Monitoring Plugin', - { tags: ['@virtualization', '@slow'] }, + 'IVT: COO - Dashboards (Perses) - Virtualization perspective', + { tags: ['@perses-dashboards', '@coo', '@virtualization', '@slow'] }, + () => { before(() => { cy.beforeBlockCOO(MCP, MP); @@ -61,7 +62,7 @@ describe('Installation: Virtualization', { tags: ['@virtualization', '@slow'] }, describe( 'IVT: COO - Dashboards (Perses) - Virtualization perspective', - { tags: ['@perses-dashboards', '@slow', '@virtualization', '@coo'] }, + { tags: ['@perses-dashboards', '@coo', '@virtualization', '@slow'] }, () => { beforeEach(() => { cy.visit('/'); diff --git a/web/cypress/support/test-tags.d.ts b/web/cypress/support/test-tags.d.ts deleted file mode 100644 index 4207b1824..000000000 --- a/web/cypress/support/test-tags.d.ts +++ /dev/null @@ -1,32 +0,0 @@ -type BasicTag = '@flaky' | '@xfail' | '@slow'; - -type HighLevelComponentTag = - | '@coo' - | '@virtualization' - | '@ols' - | '@acm-alerting' - | '@alerting' - | '@legacy-dashboards' - | '@metrics' - | '@targets' - | '@perses-dashboards' - | '@cluster-health-analyzer'; - -type SpecificFeatureTag = `@${string}-${string}`; - -type JiraTag = `@JIRA-${string}`; - -type AllowedTag = BasicTag | HighLevelComponentTag | SpecificFeatureTag | JiraTag; -type TestTags = AllowedTag | AllowedTag[]; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -declare namespace Cypress { - interface SuiteConfigOverrides { - tags?: TestTags; - } - interface TestConfigOverrides { - tags?: TestTags; - } -} - -export {}; diff --git a/web/cypress/support/test-tags.ts b/web/cypress/support/test-tags.ts new file mode 100644 index 000000000..2eb533d3e --- /dev/null +++ b/web/cypress/support/test-tags.ts @@ -0,0 +1,23 @@ +export const MODIFIER_TAGS = ['@flaky', '@xfail', '@slow'] as const; + +export const INFRASTRUCTURE_TAGS = ['@coo', '@virtualization', '@ols', '@acm'] as const; + +export const FEATURE_TAGS = [ + '@alerting', + '@legacy-dashboards', + '@metrics', + '@targets', + '@perses-dashboards', + '@cluster-health-analyzer', +] as const; + +export type ModifierTag = (typeof MODIFIER_TAGS)[number]; + +export type InfrastructureTag = (typeof INFRASTRUCTURE_TAGS)[number]; + +export type FeatureTag = (typeof FEATURE_TAGS)[number]; + +export type SpecificFeatureTag = `@${string}-${string}`; + +export type AllowedTag = ModifierTag | InfrastructureTag | FeatureTag | SpecificFeatureTag; +export type TestTags = AllowedTag | AllowedTag[]; diff --git a/web/eslint-rules/cypress-tag-order.spec.ts b/web/eslint-rules/cypress-tag-order.spec.ts new file mode 100644 index 000000000..b5510d564 --- /dev/null +++ b/web/eslint-rules/cypress-tag-order.spec.ts @@ -0,0 +1,90 @@ +import { RuleTester } from 'eslint'; +import { classifyTag, cypressTagOrder, sortTags } from './cypress-tag-order'; + +describe('classifyTag', () => { + it('classifies modifier tags', () => { + expect(classifyTag('@slow')).toBe('modifier'); + }); + + it('classifies infrastructure tags', () => { + expect(classifyTag('@acm')).toBe('infrastructure'); + expect(classifyTag('@coo')).toBe('infrastructure'); + expect(classifyTag('@virtualization')).toBe('infrastructure'); + expect(classifyTag('@ols')).toBe('infrastructure'); + }); + + it('classifies high level component tags as features', () => { + expect(classifyTag('@alerting')).toBe('feature'); + expect(classifyTag('@metrics')).toBe('feature'); + expect(classifyTag('@cluster-health-analyzer')).toBe('feature'); + }); + + it('classifies @x-y specific feature tags as features', () => { + expect(classifyTag('@perses-dev')).toBe('feature'); + }); + + it('classifies unrecognized tags as unknown', () => { + expect(classifyTag('@perses')).toBe('unknown'); + expect(classifyTag('@monitoring')).toBe('unknown'); + }); +}); + +describe('sortTags', () => { + it('sorts features, infrastructure, and modifiers alphabetically', () => { + expect(sortTags(['@slow', '@metrics', '@coo', '@alerting', '@acm'])).toEqual([ + '@alerting', + '@metrics', + '@acm', + '@coo', + '@slow', + ]); + }); + + it('places unknown tags last', () => { + expect(sortTags(['@slow', '@perses', '@alerting'])).toEqual(['@alerting', '@slow', '@perses']); + }); +}); + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: 2021, sourceType: 'module' }, +}); + +ruleTester.run('cypress-tag-order', cypressTagOrder, { + valid: [ + { + code: "describe('x', { tags: ['@alerting', '@metrics', '@acm', '@coo', '@slow'] }, () => {})", + }, + { code: "describe('x', { tags: ['@alerting'] }, () => {})" }, + { code: "describe('x', { tags: ['@slow'] }, () => {})" }, + { code: "it('x', { tags: ['@perses-dev', '@slow'] }, () => {})" }, + { code: "it('x', { tags: ['@alerting', '@coo'] }, () => {})" }, + { code: "const config = { tags: ['@slow', '@alerting'] };" }, + ], + invalid: [ + { + code: "describe('x', { tags: ['@slow', '@alerting'] }, () => {})", + output: "describe('x', { tags: ['@alerting', '@slow'] }, () => {})", + errors: [{ messageId: 'tagsNotSorted' }], + }, + { + code: "describe('x', { tags: ['@virtualization', '@alerting'] }, () => {})", + output: "describe('x', { tags: ['@alerting', '@virtualization'] }, () => {})", + errors: [{ messageId: 'tagsNotSorted' }], + }, + { + code: "describe('x', { tags: ['@slow', '@coo'] }, () => {})", + output: "describe('x', { tags: ['@coo', '@slow'] }, () => {})", + errors: [{ messageId: 'tagsNotSorted' }], + }, + { + code: "it('x', { tags: ['@slow', '@metrics'] }, () => {})", + output: "it('x', { tags: ['@metrics', '@slow'] }, () => {})", + errors: [{ messageId: 'tagsNotSorted' }], + }, + { + code: "describe('x', { tags: ['@perses', '@alerting'] }, () => {})", + output: "describe('x', { tags: ['@alerting', '@perses'] }, () => {})", + errors: [{ messageId: 'tagsNotSorted' }, { messageId: 'unknownTag' }], + }, + ], +}); diff --git a/web/eslint-rules/cypress-tag-order.ts b/web/eslint-rules/cypress-tag-order.ts new file mode 100644 index 000000000..346697e59 --- /dev/null +++ b/web/eslint-rules/cypress-tag-order.ts @@ -0,0 +1,110 @@ +import type { Rule } from 'eslint'; +import type { ArrayExpression, Property } from 'estree'; +import { FEATURE_TAGS, INFRASTRUCTURE_TAGS, MODIFIER_TAGS } from '../cypress/support/test-tags'; +import { isTagsInCallOptions, isTagsProperty } from './cypress-tags-helper'; + +const MODIFIER_SET: ReadonlySet = new Set(MODIFIER_TAGS); +const INFRASTRUCTURE_SET: ReadonlySet = new Set(INFRASTRUCTURE_TAGS); +const FEATURE_SET: ReadonlySet = new Set(FEATURE_TAGS); +const SPECIFIC_FEATURE_RE = /^@[^\s-]+-[^\s]+$/; + +export type TagKind = 'feature' | 'infrastructure' | 'modifier' | 'unknown'; + +export function classifyTag(tag: string): TagKind { + if (MODIFIER_SET.has(tag)) { + return 'modifier'; + } + if (INFRASTRUCTURE_SET.has(tag)) { + return 'infrastructure'; + } + if (FEATURE_SET.has(tag) || SPECIFIC_FEATURE_RE.test(tag)) { + return 'feature'; + } + return 'unknown'; +} + +export function sortTags(tags: string[]): string[] { + const features = tags.filter((t) => classifyTag(t) === 'feature').sort(); + const infrastructure = tags.filter((t) => classifyTag(t) === 'infrastructure').sort(); + const modifiers = tags.filter((t) => classifyTag(t) === 'modifier').sort(); + const unknowns = tags.filter((t) => classifyTag(t) === 'unknown'); + return [...features, ...infrastructure, ...modifiers, ...unknowns]; +} + +export const cypressTagOrder: Rule.RuleModule = { + meta: { + type: 'suggestion', + fixable: 'code', + docs: { + description: 'Sort Cypress tags: feature, infrastructure, then modifier tags alphabetically', + }, + schema: [], + messages: { + unknownTag: "Tag '{{tag}}' is not a recognized tag. See cypress/support/test-tags.ts.", + tagsNotSorted: + 'Cypress tags must be sorted: feature, infrastructure, then modifier tags ' + + 'alphabetically. Expected: [{{expected}}].', + }, + }, + create(context) { + const sourceCode = context.sourceCode ?? context.getSourceCode(); + + return { + Property(node: Property) { + if ( + !isTagsProperty(node) || + !isTagsInCallOptions(node) || + node.value.type !== 'ArrayExpression' + ) { + return; + } + const array = node.value as ArrayExpression; + const elements = array.elements; + + const stringElements = elements.filter( + (el): el is ArrayExpression['elements'][number] & { type: 'Literal' } => + el != null && el.type === 'Literal' && typeof el.value === 'string', + ); + if (stringElements.length !== elements.length || stringElements.length === 0) { + return; + } + + const tags = stringElements.map((el) => String((el as { value: string }).value)); + + for (const [index, tag] of tags.entries()) { + if (classifyTag(tag) === 'unknown') { + context.report({ + node: stringElements[index], + messageId: 'unknownTag', + data: { tag }, + }); + } + } + + const sorted = sortTags(tags); + if (tags.some((tag, i) => tag !== sorted[i])) { + context.report({ + node: array, + messageId: 'tagsNotSorted', + data: { expected: sorted.map((t) => `'${t}'`).join(', ') }, + fix(fixer) { + const rawByTag = new Map(); + stringElements.forEach((el, i) => { + rawByTag.set(tags[i], sourceCode.getText(el)); + }); + const used = new Map(); + const newText = sorted + .map((tag) => { + const count = used.get(tag) ?? 0; + used.set(tag, count + 1); + return rawByTag.get(tag) ?? `'${tag}'`; + }) + .join(', '); + return fixer.replaceText(array, `[${newText}]`); + }, + }); + } + }, + }; + }, +}; diff --git a/web/eslint-rules/cypress-tags-array.spec.ts b/web/eslint-rules/cypress-tags-array.spec.ts new file mode 100644 index 000000000..0d9150642 --- /dev/null +++ b/web/eslint-rules/cypress-tags-array.spec.ts @@ -0,0 +1,26 @@ +import { RuleTester } from 'eslint'; +import { cypressTagsArray } from './cypress-tags-array'; + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: 2021, sourceType: 'module' }, +}); + +ruleTester.run('cypress-tags-array', cypressTagsArray, { + valid: [ + { code: "describe('x', { tags: ['@monitoring'] }, () => {})" }, + { code: "describe('x', { tags: ['@alerts', '@slow'] }, () => {})" }, + { code: "const config = { tags: '@internal' };" }, + ], + invalid: [ + { + code: "describe('x', { tags: '@monitoring' }, () => {})", + output: "describe('x', { tags: ['@monitoring'] }, () => {})", + errors: [{ messageId: 'tagsMustBeArray' }], + }, + { + code: "it('x', { tags: '@slow' }, () => {})", + output: "it('x', { tags: ['@slow'] }, () => {})", + errors: [{ messageId: 'tagsMustBeArray' }], + }, + ], +}); diff --git a/web/eslint-rules/cypress-tags-array.ts b/web/eslint-rules/cypress-tags-array.ts new file mode 100644 index 000000000..23bb400b6 --- /dev/null +++ b/web/eslint-rules/cypress-tags-array.ts @@ -0,0 +1,41 @@ +import type { Rule } from 'eslint'; +import type { Property } from 'estree'; +import { isTagsInCallOptions, isTagsProperty } from './cypress-tags-helper'; + +export const cypressTagsArray: Rule.RuleModule = { + meta: { + type: 'suggestion', + fixable: 'code', + docs: { + description: 'Require Cypress tags to be declared as an array, even for a single tag', + }, + schema: [], + messages: { + tagsMustBeArray: 'Cypress tags must be declared as an array (e.g. tags: [{{value}}]).', + }, + }, + create(context) { + const sourceCode = context.sourceCode ?? context.getSourceCode(); + + return { + Property(node: Property) { + if ( + !isTagsProperty(node) || + !isTagsInCallOptions(node) || + node.value.type === 'ArrayExpression' + ) { + return; + } + const valueText = sourceCode.getText(node.value); + context.report({ + node: node.value, + messageId: 'tagsMustBeArray', + data: { value: valueText }, + fix(fixer) { + return fixer.replaceText(node.value, `[${valueText}]`); + }, + }); + }, + }; + }, +}; diff --git a/web/eslint-rules/cypress-tags-helper.ts b/web/eslint-rules/cypress-tags-helper.ts new file mode 100644 index 000000000..e8b597c92 --- /dev/null +++ b/web/eslint-rules/cypress-tags-helper.ts @@ -0,0 +1,43 @@ +import type { CallExpression, Expression, Node, Property, Super } from 'estree'; + +const SUPPORTED_CALLS: ReadonlySet = new Set(['describe', 'it', 'context', 'specify']); + +export function isTagsProperty(node: Property): boolean { + const key = node.key; + if (key.type === 'Identifier') { + return key.name === 'tags'; + } + if (key.type === 'Literal') { + return key.value === 'tags'; + } + return false; +} + +function calleeName(callee: Expression | Super): string | undefined { + if (callee.type === 'Identifier') { + return callee.name; + } + if (callee.type === 'MemberExpression') { + return calleeName(callee.object); + } + return undefined; +} + +function isSupportedCall(node: CallExpression): boolean { + const name = calleeName(node.callee); + return name !== undefined && SUPPORTED_CALLS.has(name); +} + +type WithParent = Node & { parent?: WithParent }; + +export function isTagsInCallOptions(node: Property): boolean { + const objectExpression = (node as WithParent).parent; + if (objectExpression?.type !== 'ObjectExpression') { + return false; + } + const call = objectExpression.parent; + if (call?.type !== 'CallExpression') { + return false; + } + return call.arguments.includes(objectExpression as never) && isSupportedCall(call); +} diff --git a/web/eslint.config.ts b/web/eslint.config.ts index 9a78fa4a2..22cb2e747 100644 --- a/web/eslint.config.ts +++ b/web/eslint.config.ts @@ -13,6 +13,8 @@ import importPlugin from 'eslint-plugin-import'; import { importBoundaryZones } from './eslint-rules/import-boundary-zones'; import { fileNaming } from './eslint-rules/file-naming'; import { requireFeatureOwners } from './eslint-rules/require-feature-owners'; +import { cypressTagOrder } from './eslint-rules/cypress-tag-order'; +import { cypressTagsArray } from './eslint-rules/cypress-tags-array'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -193,6 +195,21 @@ export default defineConfig([ 'local-rules/require-feature-owners': 'error', }, }, + { + files: ['cypress/**/*.ts', 'cypress/**/*.tsx'], + plugins: { + 'local-rules': { + rules: { + 'cypress-tag-order': cypressTagOrder, + 'cypress-tags-array': cypressTagsArray, + }, + } as any, + }, + rules: { + 'local-rules/cypress-tag-order': 'error', + 'local-rules/cypress-tags-array': 'error', + }, + }, { files: ['src/features/**/*.ts', 'src/features/**/*.tsx'], plugins: { diff --git a/web/package.json b/web/package.json index 22a4ef964..f890a8e1a 100644 --- a/web/package.json +++ b/web/package.json @@ -30,23 +30,24 @@ "test": "npm run cypress:run:ci", "test-cypress-console": "./node_modules/.bin/cypress open --browser chrome", "test-cypress-console-headless": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless", - "test-cypress-monitoring": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@flaky --@xfail --@virtualization --@coo'", - "test-cypress-monitoring-dev": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@xfail --@virtualization --@coo'", - "test-cypress-monitoring-bvt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@xfail --@virtualization --@coo'", - "test-cypress-monitoring-regression": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@flaky --@xfail --@virtualization --@coo'", - "test-cypress-alerts": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting --@flaky --@xfail'", - "test-cypress-metrics": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@metrics --@flaky --@xfail'", - "test-cypress-dashboards": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@legacy-dashboards --@flaky --@xfail'", - "test-cypress-coo": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@coo --@flaky --@xfail'", - "test-cypress-coo-bvt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@coo --@xfail'", - "test-cypress-virtualization": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@virtualization --@flaky --@xfail'", - "test-cypress-incidents": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@cluster-health-analyzer --@flaky --@xfail'", - "test-cypress-incidents-e2e": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@cluster-health-analyzer --@flaky --@xfail'", + "test-cypress-monitoring": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@flaky --@xfail --@virtualization --@coo --@acm'", + "test-cypress-monitoring-dev": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@xfail --@virtualization --@coo --@acm'", + "test-cypress-monitoring-bvt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@xfail --@virtualization --@coo --@acm'", + "test-cypress-monitoring-regression": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting @legacy-dashboards @metrics @targets --@flaky --@xfail --@acm --@coo --@virtualization'", + "test-cypress-alerts": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@alerting --@acm --@virtualization --@flaky --@xfail'", + "test-cypress-metrics": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@metrics --@virtualization --@acm --@flaky --@xfail'", + "test-cypress-dashboards": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@legacy-dashboards --@acm --@virtualization --@flaky --@xfail'", + "test-cypress-coo": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@coo --@acm --@virtualization --@flaky --@xfail'", + "test-cypress-coo-bvt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@coo --@acm --@virtualization --@xfail'", + "test-cypress-virtualization": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@virtualization --@acm --@flaky --@xfail'", + "test-cypress-incidents": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@cluster-health-analyzer --@acm --@virtualization --@flaky --@xfail'", + "test-cypress-incidents-e2e": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@cluster-health-analyzer --@acm --@virtualization --@flaky --@xfail'", "ts-node": "ts-node -O '{\"module\":\"commonjs\"}'", - "test-cypress-perses-dev": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@ols --@xfail'", - "test-cypress-perses": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@ols --@flaky --@xfail'", - "test-cypress-perses-ivt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@ols --@flaky --@xfail'", - "test-cypress-perses-ols": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@ols --@xfail'" + "test-cypress-perses-dev": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@acm --@virtualization --@ols --@xfail'", + "test-cypress-perses": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@acm --@virtualization --@ols --@flaky --@xfail'", + "test-cypress-perses-ivt": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@perses-dashboards --@acm --@virtualization --@ols --@flaky --@xfail'", + "test-cypress-perses-ols": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@ols --@acm --@xfail'", + "test-cypress-acm": "node --max-old-space-size=4096 ./node_modules/.bin/cypress run --browser chrome --headless --env grepTags='@acm --@xfail'" }, "dependencies": { "@codemirror/autocomplete": "^6.0.4",