From bc7c74a98f8b24adefee36dbb65903dcacc6d799 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 13 Aug 2026 00:44:27 +1200 Subject: [PATCH 1/2] Normalize encoded URL paths. --- lib/protocol/url/path.rb | 88 +++++++++++++++++++++++++++--- lib/protocol/url/relative.rb | 10 ++-- releases.md | 4 ++ test/protocol/url/path.rb | 100 +++++++++++++++++++++++++++++++++- test/protocol/url/relative.rb | 20 +++++-- 5 files changed, 202 insertions(+), 20 deletions(-) diff --git a/lib/protocol/url/path.rb b/lib/protocol/url/path.rb index 6339135..47da670 100644 --- a/lib/protocol/url/path.rb +++ b/lib/protocol/url/path.rb @@ -20,7 +20,8 @@ class Path EMPTY_SEGMENTS = [].freeze ROOT_SEGMENTS = ["", ""].freeze - private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS + NORMALIZATION_PATTERN = /%[0-9A-Fa-f]{2}|%|[^a-zA-Z0-9_.~!$&'()*+,;=:@-]/ + private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS, :NORMALIZATION_PATTERN # Coerce an encoded string or encoded segment array into a path. # @@ -252,7 +253,43 @@ def local_path(root) alias to_s encoded alias to_str encoded - # Simplify this path in place by resolving literal or percent-encoded dot segments and repeated separators. + # Normalize the encoded spelling of this path. + # + # Percent-encoded unreserved characters are decoded, retained percent escapes + # use uppercase hexadecimal digits, and literal characters outside the path + # segment grammar are percent encoded. Reserved characters retain their + # encoded or literal form because those forms are not generally equivalent. + # + # This operation preserves the path structure. Use {simplify} separately when + # application semantics permit resolving dot segments or repeated separators. + # + # @returns [Path] The normalized path, or this path if already normalized. + # @raises [ArgumentError] If the path contains malformed percent encoding, NUL, or invalid string encoding. + def normalize + encoded = self.encoded + unless encoded.valid_encoding? && encoded.encoding.ascii_compatible? + raise ArgumentError, "Path segment has invalid encoding!" + end + + segments = self.segments + normalized_segments = nil + + segments.each_with_index do |segment, index| + next unless NORMALIZATION_PATTERN.match?(segment) + + normalized = normalize_segment(segment) + next if normalized == segment + + normalized_segments ||= segments.dup + normalized_segments[index] = normalized + end + + return self unless normalized_segments + + return self.class.new(nil, normalized_segments) + end + + # Simplify this path in place by resolving literal or percent-encoded dot segments. # # @returns [Path | Nil] This path when changed, otherwise `nil`. def simplify! @@ -265,7 +302,7 @@ def simplify! return self end - # Return a canonical path by resolving literal or percent-encoded dot segments and repeated separators. + # Return a canonical path by resolving literal or percent-encoded dot segments. # # Absolute paths do not retain parent components above the root. Relative paths # retain leading parent components which cannot be resolved locally. @@ -342,6 +379,42 @@ def relative(from) private + # Normalize one encoded path segment: + def normalize_segment(segment) + return segment.gsub(NORMALIZATION_PATTERN) do |character| + byte = character.getbyte(0) + + if byte == 0 + raise ArgumentError, "Path segment contains NUL!" + elsif byte == 0x25 + if character.bytesize == 1 + raise ArgumentError, "String contains malformed percent encoding!" + end + + byte = character.byteslice(1, 2).to_i(16) + if byte == 0 + raise ArgumentError, "Path segment contains NUL!" + elsif unreserved_byte?(byte) + byte.chr + else + character.upcase + end + else + Encoding.escape(character) + end + end + end + + # Whether the byte represents an unreserved URI character: + def unreserved_byte?(byte) + case byte + when 0x30..0x39, 0x41..0x5A, 0x61..0x7A, 0x2D, 0x2E, 0x5F, 0x7E + return true + else + return false + end + end + # Identify dot segments, including percent-encoded spellings. RFC 3986 treats # percent-encoded unreserved characters as equivalent to their literal forms; # the WHATWG URL Standard explicitly recognizes `%2e`, `.%2e`, `%2e.`, and @@ -370,8 +443,7 @@ def simplification_index(segments) if dot == "." return index elsif segment == "" - # Leading and trailing empty components are significant. - return index if index > 0 && index < last_index + # Empty segments are significant and do not require simplification. elsif dot == ".." # Absolute paths cannot retain parent components. Relative paths # can retain them only before the first regular component. @@ -414,11 +486,9 @@ def simplify_segments!(segments, start_index = nil) segments[offset] = "" offset += 1 end - elsif segment == "" && index != last_index - # Collapse repeated separators. elsif dot == ".." && offset > 0 && dot_segment(segments[offset - 1]) != ".." - # Pop a component, but never pop the absolute-path root. - offset -= 1 if segments[offset - 1] != "" + # Pop a component, but never pop the leading absolute-path root: + offset -= 1 unless segments.first == "" && offset == 1 # A trailing parent reference also denotes a directory. if index == last_index diff --git a/lib/protocol/url/relative.rb b/lib/protocol/url/relative.rb index c671aab..48495d3 100644 --- a/lib/protocol/url/relative.rb +++ b/lib/protocol/url/relative.rb @@ -132,12 +132,14 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true) self.class.new(path || @path, query, fragment) end - # Normalize the path by resolving "." and ".." segments and removing duplicate slashes. + # Normalize the encoded path and simplify its structure. # - # This modifies the URL in-place by simplifying the path component: + # This modifies the URL in-place by normalizing and simplifying the path component: + # - Decodes percent-encoded unreserved characters + # - Uses uppercase hexadecimal digits for retained percent escapes # - Removes "." segments (current directory) # - Resolves ".." segments (parent directory) - # - Collapses multiple consecutive slashes to single slashes (except at start) + # - Preserves empty path segments represented by consecutive slashes # # @returns [self] The normalized URL. # @@ -146,7 +148,7 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true) # url.normalize! # url.path.to_s # => "/foo/bar/qux" def normalize! - @path = @path.simplify + @path = @path.normalize.simplify return self end diff --git a/releases.md b/releases.md index b6a5e17..e051a88 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add conservative normalization of encoded URL paths. + ## v0.12.0 - Allow unfrozen relative and absolute URLs to replace their components. diff --git a/test/protocol/url/path.rb b/test/protocol/url/path.rb index 8ddb623..5d5fdaa 100644 --- a/test/protocol/url/path.rb +++ b/test/protocol/url/path.rb @@ -265,7 +265,7 @@ "removes a leading current directory" => [[".", "a", "b"], ["a", "b"]], "removes an intermediate current directory" => [["a", ".", "b"], ["a", "b"]], "preserves a trailing directory marker" => [["a", "b", "."], ["a", "b", ""]], - "collapses repeated separators" => [["a", "", "b", "", "", "c"], ["a", "b", "c"]], + "preserves repeated separators" => [["a", "", "b", "", "", "c"], ["a", "", "b", "", "", "c"]], "preserves a trailing separator" => [["a", "b", ""], ["a", "b", ""]], "resolves a parent directory" => [["a", "b", "..", "c"], ["a", "c"]], "resolves multiple parent directories" => [["a", "b", "c", "..", "..", "d"], ["a", "d"]], @@ -275,7 +275,7 @@ "preserves a parent at the relative root" => [["..", "a"], ["..", "a"]], "preserves multiple parents at the relative root" => [["..", "..", "a"], ["..", "..", "a"]], "retains unresolved parent markers" => [["a", "..", "..", "b"], ["..", "b"]], - "handles a complex path" => [["", "a", "b", ".", "c", "..", "d", "", "e"], ["", "a", "b", "d", "e"]], + "handles a complex path" => [["", "a", "b", ".", "c", "..", "d", "", "e"], ["", "a", "b", "d", "", "e"]], "resolves all dot segments" => [[".", "a", ".", "b", "..", "c", ".", "d", ".."], ["a", "c", ""]], }.each do |description, (components, expected)| it description do @@ -309,6 +309,94 @@ expect(path.simplify.encoded).to be == "/a/c" end + + it "resolves a parent following an empty segment" do + path = Protocol::URL::Path["/a//../b"] + + expect(path.simplify.encoded).to be == "/a/b" + end + end + + with "#normalize" do + it "decodes percent-encoded unreserved characters" do + path = Protocol::URL::Path["/%41%7a%30%2d%2e%5f%7e"] + + expect(path.normalize.encoded).to be == "/Az0-._~" + end + + it "uses uppercase hexadecimal digits for retained percent escapes" do + path = Protocol::URL::Path["/a%2fb%3fc%ff"] + + expect(path.normalize.encoded).to be == "/a%2Fb%3Fc%FF" + end + + it "preserves literal path segment delimiters" do + path = Protocol::URL::Path["/!$&'()*+,;=:@"] + + expect(path.normalize).to be_equal(path) + end + + it "preserves the distinction between encoded and literal reserved characters" do + path = Protocol::URL::Path["/a%3Ab:a%2Fb"] + + expect(path.normalize).to be_equal(path) + end + + it "percent encodes characters outside the path segment grammar" do + path = Protocol::URL::Path["/hello world?[x]#"] + + expect(path.normalize.encoded).to be == "/hello%20world%3F%5Bx%5D%23" + end + + it "percent encodes literal unicode characters" do + path = Protocol::URL::Path["/❤️"] + + expect(path.normalize.encoded).to be == "/%E2%9D%A4%EF%B8%8F" + end + + it "preserves path structure" do + path = Protocol::URL::Path["//a/%2e%2e/%77elcome"] + + expect(path.normalize.encoded).to be == "//a/../welcome" + end + + it "returns itself when already normalized" do + path = Protocol::URL::Path["/welcome/a:b/%2F"] + + expect(path.normalize).to be_equal(path) + end + + it "rejects literal NUL" do + path = Protocol::URL::Path["/a\0b"] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!") + end + + it "rejects percent-encoded NUL" do + path = Protocol::URL::Path["/a%00b"] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!") + end + + it "rejects malformed percent encoding" do + ["/a%", "/a%0", "/a%gg"].each do |encoded| + expect do + Protocol::URL::Path[encoded].normalize + end.to raise_exception(ArgumentError, message: be == "String contains malformed percent encoding!") + end + end + + it "rejects invalid string encoding" do + path = Protocol::URL::Path["/a\xFF".dup.force_encoding(::Encoding::UTF_8)] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment has invalid encoding!") + end end with "#simplify!" do @@ -336,7 +424,7 @@ path = Protocol::URL::Path[["", "..", "a", "", "b", "..", ""]] path.simplify! - expect(path.components).to be == ["", "a", ""] + expect(path.components).to be == ["", "a", "", ""] end end @@ -444,6 +532,12 @@ expect(Protocol::URL::Path["documents/report.pdf"].local_path(root)).to be == expected end + it "maps empty URL segments to the same local filesystem path" do + expected = File.join(root, "documents", "report.pdf") + + expect(Protocol::URL::Path["/documents//report.pdf"].local_path(root)).to be == expected + end + it "unescapes percent-encoded and Unicode characters" do expect(Protocol::URL::Path["/files/My%20Document.txt"].local_path(root)).to be == File.join(root, "files", "My Document.txt") expect(Protocol::URL::Path["/files/%E2%9D%A4%EF%B8%8F.txt"].local_path(root)).to be == File.join(root, "files", "❤️.txt") diff --git a/test/protocol/url/relative.rb b/test/protocol/url/relative.rb index 0c652bd..e903bcc 100644 --- a/test/protocol/url/relative.rb +++ b/test/protocol/url/relative.rb @@ -269,6 +269,18 @@ end with "#normalize!" do + it "normalizes the encoded path" do + url = Protocol::URL::Relative.new("/%66oo/a%2fb") + url.normalize! + expect(url.path).to be == Protocol::URL::Path["/foo/a%2Fb"] + end + + it "simplifies normalized dot segments" do + url = Protocol::URL::Relative.new("/foo/%2e%2e/bar") + url.normalize! + expect(url.path).to be == Protocol::URL::Path["/bar"] + end + it "removes dot segments" do url = Protocol::URL::Relative.new("/foo/./bar") url.normalize! @@ -281,16 +293,16 @@ expect(url.path).to be == Protocol::URL::Path["/foo/baz"] end - it "collapses multiple slashes" do + it "preserves empty path segments" do url = Protocol::URL::Relative.new("/foo//bar///baz") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo/bar/baz"] + expect(url.path).to be == Protocol::URL::Path["/foo//bar///baz"] end it "handles complex paths" do url = Protocol::URL::Relative.new("/foo//bar/./baz/../qux") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo/bar/qux"] + expect(url.path).to be == Protocol::URL::Path["/foo//bar/qux"] end it "preserves trailing slash" do @@ -314,7 +326,7 @@ it "preserves query and fragment" do url = Protocol::URL::Relative.new("/foo//bar", "q=test", "section") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo/bar"] + expect(url.path).to be == Protocol::URL::Path["/foo//bar"] expect(url.query).to be == "q=test" expect(url.fragment).to be == "section" end From e21d186b52e55e2e4cf1bbd9dc135dab11ee3207 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 13 Aug 2026 13:25:41 +1200 Subject: [PATCH 2/2] Document lossy path normalization. --- guides/getting-started/readme.md | 3 +++ lib/protocol/url/path.rb | 15 +++++++++------ lib/protocol/url/relative.rb | 5 ++++- test/protocol/url/path.rb | 10 +++++----- test/protocol/url/relative.rb | 8 ++++---- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 1453a4b..8568a2c 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -290,11 +290,14 @@ messy = Protocol::URL["https://example.com/a/b/../c/./d"] # Parsing preserves the original path until simplification is requested: messy.path.to_s # => "/a/b/../c/./d" +# Normalization is intentionally lossy and produces a canonical path: messy.normalize! messy.path.to_s # => "/a/c/d" messy.to_s # => "https://example.com/a/c/d" ``` +If the original path structure is significant, retain the parsed URL and do not call `normalize!`. + ## Best Practices ### Choose the Right Class diff --git a/lib/protocol/url/path.rb b/lib/protocol/url/path.rb index 47da670..4fdc179 100644 --- a/lib/protocol/url/path.rb +++ b/lib/protocol/url/path.rb @@ -261,7 +261,7 @@ def local_path(root) # encoded or literal form because those forms are not generally equivalent. # # This operation preserves the path structure. Use {simplify} separately when - # application semantics permit resolving dot segments or repeated separators. + # application semantics permit resolving dot segments or collapsing repeated separators. # # @returns [Path] The normalized path, or this path if already normalized. # @raises [ArgumentError] If the path contains malformed percent encoding, NUL, or invalid string encoding. @@ -289,7 +289,7 @@ def normalize return self.class.new(nil, normalized_segments) end - # Simplify this path in place by resolving literal or percent-encoded dot segments. + # Simplify this path in place by resolving literal or percent-encoded dot segments and repeated separators. # # @returns [Path | Nil] This path when changed, otherwise `nil`. def simplify! @@ -302,7 +302,7 @@ def simplify! return self end - # Return a canonical path by resolving literal or percent-encoded dot segments. + # Return a canonical path by resolving literal or percent-encoded dot segments and repeated separators. # # Absolute paths do not retain parent components above the root. Relative paths # retain leading parent components which cannot be resolved locally. @@ -443,7 +443,8 @@ def simplification_index(segments) if dot == "." return index elsif segment == "" - # Empty segments are significant and do not require simplification. + # Leading and trailing empty components are significant. + return index if index > 0 && index < last_index elsif dot == ".." # Absolute paths cannot retain parent components. Relative paths # can retain them only before the first regular component. @@ -486,9 +487,11 @@ def simplify_segments!(segments, start_index = nil) segments[offset] = "" offset += 1 end + elsif segment == "" && index != last_index + # Collapse repeated separators: elsif dot == ".." && offset > 0 && dot_segment(segments[offset - 1]) != ".." - # Pop a component, but never pop the leading absolute-path root: - offset -= 1 unless segments.first == "" && offset == 1 + # Pop a component, but never pop the absolute-path root: + offset -= 1 if segments[offset - 1] != "" # A trailing parent reference also denotes a directory. if index == last_index diff --git a/lib/protocol/url/relative.rb b/lib/protocol/url/relative.rb index 48495d3..33cb165 100644 --- a/lib/protocol/url/relative.rb +++ b/lib/protocol/url/relative.rb @@ -139,7 +139,10 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true) # - Uses uppercase hexadecimal digits for retained percent escapes # - Removes "." segments (current directory) # - Resolves ".." segments (parent directory) - # - Preserves empty path segments represented by consecutive slashes + # - Collapses empty path segments represented by consecutive slashes + # + # Normalization is intentionally lossy. Callers that need to preserve the + # original path structure should retain the parsed URL and avoid this method. # # @returns [self] The normalized URL. # diff --git a/test/protocol/url/path.rb b/test/protocol/url/path.rb index 5d5fdaa..4001414 100644 --- a/test/protocol/url/path.rb +++ b/test/protocol/url/path.rb @@ -265,7 +265,7 @@ "removes a leading current directory" => [[".", "a", "b"], ["a", "b"]], "removes an intermediate current directory" => [["a", ".", "b"], ["a", "b"]], "preserves a trailing directory marker" => [["a", "b", "."], ["a", "b", ""]], - "preserves repeated separators" => [["a", "", "b", "", "", "c"], ["a", "", "b", "", "", "c"]], + "collapses repeated separators" => [["a", "", "b", "", "", "c"], ["a", "b", "c"]], "preserves a trailing separator" => [["a", "b", ""], ["a", "b", ""]], "resolves a parent directory" => [["a", "b", "..", "c"], ["a", "c"]], "resolves multiple parent directories" => [["a", "b", "c", "..", "..", "d"], ["a", "d"]], @@ -275,7 +275,7 @@ "preserves a parent at the relative root" => [["..", "a"], ["..", "a"]], "preserves multiple parents at the relative root" => [["..", "..", "a"], ["..", "..", "a"]], "retains unresolved parent markers" => [["a", "..", "..", "b"], ["..", "b"]], - "handles a complex path" => [["", "a", "b", ".", "c", "..", "d", "", "e"], ["", "a", "b", "d", "", "e"]], + "handles a complex path" => [["", "a", "b", ".", "c", "..", "d", "", "e"], ["", "a", "b", "d", "e"]], "resolves all dot segments" => [[".", "a", ".", "b", "..", "c", ".", "d", ".."], ["a", "c", ""]], }.each do |description, (components, expected)| it description do @@ -310,10 +310,10 @@ expect(path.simplify.encoded).to be == "/a/c" end - it "resolves a parent following an empty segment" do + it "resolves a parent following a repeated separator" do path = Protocol::URL::Path["/a//../b"] - expect(path.simplify.encoded).to be == "/a/b" + expect(path.simplify.encoded).to be == "/b" end end @@ -424,7 +424,7 @@ path = Protocol::URL::Path[["", "..", "a", "", "b", "..", ""]] path.simplify! - expect(path.components).to be == ["", "a", "", ""] + expect(path.components).to be == ["", "a", ""] end end diff --git a/test/protocol/url/relative.rb b/test/protocol/url/relative.rb index e903bcc..980349c 100644 --- a/test/protocol/url/relative.rb +++ b/test/protocol/url/relative.rb @@ -293,16 +293,16 @@ expect(url.path).to be == Protocol::URL::Path["/foo/baz"] end - it "preserves empty path segments" do + it "collapses empty path segments" do url = Protocol::URL::Relative.new("/foo//bar///baz") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo//bar///baz"] + expect(url.path).to be == Protocol::URL::Path["/foo/bar/baz"] end it "handles complex paths" do url = Protocol::URL::Relative.new("/foo//bar/./baz/../qux") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo//bar/qux"] + expect(url.path).to be == Protocol::URL::Path["/foo/bar/qux"] end it "preserves trailing slash" do @@ -326,7 +326,7 @@ it "preserves query and fragment" do url = Protocol::URL::Relative.new("/foo//bar", "q=test", "section") url.normalize! - expect(url.path).to be == Protocol::URL::Path["/foo//bar"] + expect(url.path).to be == Protocol::URL::Path["/foo/bar"] expect(url.query).to be == "q=test" expect(url.fragment).to be == "section" end