Normalize HTML when comparing featured item content (unblocks test-functional on all PRs)

test/functional/collections/UpdateCollectionFeaturedItems.test.ts

@@ -27,6 +27,7 @@ import {
   FeaturedItemType
 } from '../../../src/collections/domain/models/FeaturedItem'
 import { uploadFileViaApi } from '../../testHelpers/files/filesHelper'
+import { normalizeHtml } from '../../testHelpers/html/htmlNormalizer'
 import {
   deletePublishedDatasetViaApi,
   publishDatasetViaApi,
@@ -165,7 +166,9 @@ describe('execute', () => {
     expect(secondItemResponse.imageFileUrl).toBeUndefined()
     expect(secondItemResponse.imageFileName).toBeUndefined()
 
-    expect(thirdItemResponse.content).toEqual(EXPECTED_CONTENT_FIELD_WITH_ALL_TAGS)
+    expect(normalizeHtml(thirdItemResponse.content)).toEqual(
+      normalizeHtml(EXPECTED_CONTENT_FIELD_WITH_ALL_TAGS)
+    )
     expect(thirdItemResponse.displayOrder).toBe(newFeaturedItems[2].displayOrder)
     expect(thirdItemResponse.imageFileName).toEqual('featured-item-test-image-3.png')
     expect(thirdItemResponse.imageFileUrl).toContain(

test/testHelpers/html/htmlNormalizer.ts

@@ -0,0 +1,145 @@
+const WHITESPACE_SENSITIVE_TAGS = new Set(['pre', 'textarea'])
+
+const BLOCK_TAGS = new Set([
+  'address',
+  'article',
+  'aside',
+  'blockquote',
+  'body',
+  'br',
+  'div',
+  'dd',
+  'dl',
+  'dt',
+  'fieldset',
+  'figcaption',
+  'figure',
+  'footer',
+  'form',
+  'h1',
+  'h2',
+  'h3',
+  'h4',
+  'h5',
+  'h6',
+  'head',
+  'header',
+  'hr',
+  'html',
+  'li',
+  'main',
+  'nav',
+  'ol',
+  'p',
+  'pre',
+  'section',
+  'table',
+  'tbody',
+  'td',
+  'tfoot',
+  'th',
+  'thead',
+  'tr',
+  'ul'
+])
+
+const TAG_PATTERN = /^<\s*(\/?)\s*([a-zA-Z][\w:-]*)([\s\S]*?)(\/?)\s*>$/
+const ATTRIBUTE_PATTERN = /([\w:-]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+))?/g
+
+interface ParsedTag {
+  closing: boolean
+  name: string
+  selfClosing: boolean
+}
+
+const parseTag = (token: string): ParsedTag | undefined => {
+  const match = TAG_PATTERN.exec(token)
+  if (match === null) {
+    return undefined
+  }
+  return {
+    closing: match[1] === '/',
+    name: match[2].toLowerCase(),
+    selfClosing: match[4] === '/'
+  }
+}
+
+const normalizeTag = (token: string): string => {
+  const match = TAG_PATTERN.exec(token)
+  if (match === null) {
+    return token
+  }
+  const [, closing, name, attributeSource, selfClosing] = match
+  const attributes = Array.from(attributeSource.matchAll(ATTRIBUTE_PATTERN))
+    .map(([, attributeName, attributeValue]) =>
+      attributeValue === undefined
+        ? attributeName.toLowerCase()
+        : `${attributeName.toLowerCase()}=${normalizeAttributeValue(attributeValue)}`
+    )
+    .sort()
+  const renderedAttributes = attributes.length === 0 ? '' : ` ${attributes.join(' ')}`
+  return `<${closing}${name.toLowerCase()}${renderedAttributes}${selfClosing}>`
+}
+
+const normalizeAttributeValue = (value: string): string => {
+  const unquoted =
+    (value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))
+      ? value.slice(1, -1)
+      : value
+  return `"${unquoted}"`
+}
+
+const isBlockBoundary = (token: string | undefined): boolean => {
+  if (token === undefined) {
+    return true
+  }
+  const tag = parseTag(token)
+  return tag !== undefined && BLOCK_TAGS.has(tag.name)
+}
+
+export const normalizeHtml = (html: string): string => {
+  const tokens = html.split(/(<[^>]*>)/).filter((token) => token !== '')
+  const normalized: string[] = []
+  let whitespaceSensitiveDepth = 0
+
+  tokens.forEach((token, index) => {
+    const tag = token.startsWith('<') ? parseTag(token) : undefined
+
+    if (tag !== undefined) {
+      if (tag.closing && WHITESPACE_SENSITIVE_TAGS.has(tag.name)) {
+        whitespaceSensitiveDepth = Math.max(0, whitespaceSensitiveDepth - 1)
+      }
+      normalized.push(normalizeTag(token))
+      if (!tag.closing && !tag.selfClosing && WHITESPACE_SENSITIVE_TAGS.has(tag.name)) {
+        whitespaceSensitiveDepth += 1
+      }
+      return
+    }
+
+    if (token.startsWith('<') || whitespaceSensitiveDepth > 0) {
+      normalized.push(token)
+      return
+    }
+
+    const previousToken = tokens[index - 1]
+    const nextToken = tokens[index + 1]
+
+    if (token.trim() === '') {
+      if (!isBlockBoundary(previousToken) && !isBlockBoundary(nextToken)) {
+        normalized.push(' ')
+      }
+      return
+    }
+
+    let text = token.replace(/\s+/g, ' ')
+    if (isBlockBoundary(previousToken)) {
+      text = text.replace(/^ /, '')
+    }
+    if (isBlockBoundary(nextToken)) {
+      text = text.replace(/ $/, '')
+    }
+    normalized.push(text)
+  })
+
+  return normalized.join('')
+}

test/unit/testHelpers/htmlNormalizer.test.ts

@@ -0,0 +1,77 @@
+import { normalizeHtml } from '../../testHelpers/html/htmlNormalizer'
+import {
+  CONTENT_FIELD_WITH_ALL_TAGS,
+  EXPECTED_CONTENT_FIELD_WITH_ALL_TAGS
+} from '../../testHelpers/collections/collectionHelper'
+
+describe('normalizeHtml', () => {
+  describe('differences the server may introduce', () => {
+    test('should ignore the order of attributes', () => {
+      expect(
+        normalizeHtml('<a target="_blank" rel="nofollow" class="x" href="https://a.b">t</a>')
+      ).toEqual(
+        normalizeHtml('<a class="x" href="https://a.b" rel="nofollow" target="_blank">t</a>')
+      )
+    })
+
+    test('should ignore indentation introduced between block elements', () => {
+      expect(normalizeHtml('<ul><li><p>Item</p></li></ul>')).toEqual(
+        normalizeHtml('<ul>\n <li>\n  <p>Item</p>\n </li>\n</ul>')
+      )
+    })
+
+    test('should ignore indentation around the content of a block element', () => {
+      expect(normalizeHtml('<p>Item</p>')).toEqual(normalizeHtml('<p>\n  Item\n</p>'))
+    })
+
+    test('should ignore the case of tag and attribute names', () => {
+      expect(normalizeHtml('<P CLASS="x">t</P>')).toEqual(normalizeHtml('<p class="x">t</p>'))
+    })
+
+    test('should treat the sent and pretty-printed forms of the featured item fixture as equal', () => {
+      expect(normalizeHtml(CONTENT_FIELD_WITH_ALL_TAGS)).toEqual(
+        normalizeHtml(EXPECTED_CONTENT_FIELD_WITH_ALL_TAGS)
+      )
+    })
+  })
+
+  describe('differences that must still be detected', () => {
+    test('should not ignore differing text content', () => {
+      expect(normalizeHtml('<p>Item</p>')).not.toEqual(normalizeHtml('<p>Other</p>'))
+    })
+
+    test('should not ignore differing attribute values', () => {
+      expect(normalizeHtml('<a href="https://a.b">t</a>')).not.toEqual(
+        normalizeHtml('<a href="https://evil.example">t</a>')
+      )
+    })
+
+    test('should not ignore a dropped attribute', () => {
+      expect(normalizeHtml('<a rel="nofollow" href="https://a.b">t</a>')).not.toEqual(
+        normalizeHtml('<a href="https://a.b">t</a>')
+      )
+    })
+
+    test('should not ignore differing structure', () => {
+      expect(normalizeHtml('<ul><li>a</li><li>b</li></ul>')).not.toEqual(
+        normalizeHtml('<ul><li>a</li></ul>')
+      )
+    })
+
+    test('should not ignore a changed tag', () => {
+      expect(normalizeHtml('<strong>t</strong>')).not.toEqual(normalizeHtml('<em>t</em>'))
+    })
+
+    test('should preserve whitespace inside a preformatted block', () => {
+      expect(normalizeHtml('<pre><code>  indented\n  lines</code></pre>')).not.toEqual(
+        normalizeHtml('<pre><code>indented lines</code></pre>')
+      )
+    })
+
+    test('should preserve whitespace that separates inline elements', () => {
+      expect(normalizeHtml('<p><em>a</em> <em>b</em></p>')).not.toEqual(
+        normalizeHtml('<p><em>a</em><em>b</em></p>')
+      )
+    })
+  })
+})