diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 1453a4b..8568a2c 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -290,11 +290,14 @@ messy = Protocol::URL["https://example.com/a/b/../c/./d"] # Parsing preserves the original path until simplification is requested: messy.path.to_s # => "/a/b/../c/./d" +# Normalization is intentionally lossy and produces a canonical path: messy.normalize! messy.path.to_s # => "/a/c/d" messy.to_s # => "https://example.com/a/c/d" ``` +If the original path structure is significant, retain the parsed URL and do not call `normalize!`. + ## Best Practices ### Choose the Right Class diff --git a/lib/protocol/url/path.rb b/lib/protocol/url/path.rb index 6339135..4fdc179 100644 --- a/lib/protocol/url/path.rb +++ b/lib/protocol/url/path.rb @@ -20,7 +20,8 @@ class Path EMPTY_SEGMENTS = [].freeze ROOT_SEGMENTS = ["", ""].freeze - private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS + NORMALIZATION_PATTERN = /%[0-9A-Fa-f]{2}|%|[^a-zA-Z0-9_.~!$&'()*+,;=:@-]/ + private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS, :NORMALIZATION_PATTERN # Coerce an encoded string or encoded segment array into a path. # @@ -252,6 +253,42 @@ def local_path(root) alias to_s encoded alias to_str encoded + # Normalize the encoded spelling of this path. + # + # Percent-encoded unreserved characters are decoded, retained percent escapes + # use uppercase hexadecimal digits, and literal characters outside the path + # segment grammar are percent encoded. Reserved characters retain their + # encoded or literal form because those forms are not generally equivalent. + # + # This operation preserves the path structure. Use {simplify} separately when + # application semantics permit resolving dot segments or collapsing repeated separators. + # + # @returns [Path] The normalized path, or this path if already normalized. + # @raises [ArgumentError] If the path contains malformed percent encoding, NUL, or invalid string encoding. + def normalize + encoded = self.encoded + unless encoded.valid_encoding? && encoded.encoding.ascii_compatible? + raise ArgumentError, "Path segment has invalid encoding!" + end + + segments = self.segments + normalized_segments = nil + + segments.each_with_index do |segment, index| + next unless NORMALIZATION_PATTERN.match?(segment) + + normalized = normalize_segment(segment) + next if normalized == segment + + normalized_segments ||= segments.dup + normalized_segments[index] = normalized + end + + return self unless normalized_segments + + return self.class.new(nil, normalized_segments) + end + # Simplify this path in place by resolving literal or percent-encoded dot segments and repeated separators. # # @returns [Path | Nil] This path when changed, otherwise `nil`. @@ -342,6 +379,42 @@ def relative(from) private + # Normalize one encoded path segment: + def normalize_segment(segment) + return segment.gsub(NORMALIZATION_PATTERN) do |character| + byte = character.getbyte(0) + + if byte == 0 + raise ArgumentError, "Path segment contains NUL!" + elsif byte == 0x25 + if character.bytesize == 1 + raise ArgumentError, "String contains malformed percent encoding!" + end + + byte = character.byteslice(1, 2).to_i(16) + if byte == 0 + raise ArgumentError, "Path segment contains NUL!" + elsif unreserved_byte?(byte) + byte.chr + else + character.upcase + end + else + Encoding.escape(character) + end + end + end + + # Whether the byte represents an unreserved URI character: + def unreserved_byte?(byte) + case byte + when 0x30..0x39, 0x41..0x5A, 0x61..0x7A, 0x2D, 0x2E, 0x5F, 0x7E + return true + else + return false + end + end + # Identify dot segments, including percent-encoded spellings. RFC 3986 treats # percent-encoded unreserved characters as equivalent to their literal forms; # the WHATWG URL Standard explicitly recognizes `%2e`, `.%2e`, `%2e.`, and @@ -415,9 +488,9 @@ def simplify_segments!(segments, start_index = nil) offset += 1 end elsif segment == "" && index != last_index - # Collapse repeated separators. + # Collapse repeated separators: elsif dot == ".." && offset > 0 && dot_segment(segments[offset - 1]) != ".." - # Pop a component, but never pop the absolute-path root. + # Pop a component, but never pop the absolute-path root: offset -= 1 if segments[offset - 1] != "" # A trailing parent reference also denotes a directory. diff --git a/lib/protocol/url/relative.rb b/lib/protocol/url/relative.rb index c671aab..33cb165 100644 --- a/lib/protocol/url/relative.rb +++ b/lib/protocol/url/relative.rb @@ -132,12 +132,17 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true) self.class.new(path || @path, query, fragment) end - # Normalize the path by resolving "." and ".." segments and removing duplicate slashes. + # Normalize the encoded path and simplify its structure. # - # This modifies the URL in-place by simplifying the path component: + # This modifies the URL in-place by normalizing and simplifying the path component: + # - Decodes percent-encoded unreserved characters + # - Uses uppercase hexadecimal digits for retained percent escapes # - Removes "." segments (current directory) # - Resolves ".." segments (parent directory) - # - Collapses multiple consecutive slashes to single slashes (except at start) + # - Collapses empty path segments represented by consecutive slashes + # + # Normalization is intentionally lossy. Callers that need to preserve the + # original path structure should retain the parsed URL and avoid this method. # # @returns [self] The normalized URL. # @@ -146,7 +151,7 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true) # url.normalize! # url.path.to_s # => "/foo/bar/qux" def normalize! - @path = @path.simplify + @path = @path.normalize.simplify return self end diff --git a/releases.md b/releases.md index b6a5e17..e051a88 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add conservative normalization of encoded URL paths. + ## v0.12.0 - Allow unfrozen relative and absolute URLs to replace their components. diff --git a/test/protocol/url/path.rb b/test/protocol/url/path.rb index 8ddb623..4001414 100644 --- a/test/protocol/url/path.rb +++ b/test/protocol/url/path.rb @@ -309,6 +309,94 @@ expect(path.simplify.encoded).to be == "/a/c" end + + it "resolves a parent following a repeated separator" do + path = Protocol::URL::Path["/a//../b"] + + expect(path.simplify.encoded).to be == "/b" + end + end + + with "#normalize" do + it "decodes percent-encoded unreserved characters" do + path = Protocol::URL::Path["/%41%7a%30%2d%2e%5f%7e"] + + expect(path.normalize.encoded).to be == "/Az0-._~" + end + + it "uses uppercase hexadecimal digits for retained percent escapes" do + path = Protocol::URL::Path["/a%2fb%3fc%ff"] + + expect(path.normalize.encoded).to be == "/a%2Fb%3Fc%FF" + end + + it "preserves literal path segment delimiters" do + path = Protocol::URL::Path["/!$&'()*+,;=:@"] + + expect(path.normalize).to be_equal(path) + end + + it "preserves the distinction between encoded and literal reserved characters" do + path = Protocol::URL::Path["/a%3Ab:a%2Fb"] + + expect(path.normalize).to be_equal(path) + end + + it "percent encodes characters outside the path segment grammar" do + path = Protocol::URL::Path["/hello world?[x]#"] + + expect(path.normalize.encoded).to be == "/hello%20world%3F%5Bx%5D%23" + end + + it "percent encodes literal unicode characters" do + path = Protocol::URL::Path["/❤️"] + + expect(path.normalize.encoded).to be == "/%E2%9D%A4%EF%B8%8F" + end + + it "preserves path structure" do + path = Protocol::URL::Path["//a/%2e%2e/%77elcome"] + + expect(path.normalize.encoded).to be == "//a/../welcome" + end + + it "returns itself when already normalized" do + path = Protocol::URL::Path["/welcome/a:b/%2F"] + + expect(path.normalize).to be_equal(path) + end + + it "rejects literal NUL" do + path = Protocol::URL::Path["/a\0b"] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!") + end + + it "rejects percent-encoded NUL" do + path = Protocol::URL::Path["/a%00b"] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!") + end + + it "rejects malformed percent encoding" do + ["/a%", "/a%0", "/a%gg"].each do |encoded| + expect do + Protocol::URL::Path[encoded].normalize + end.to raise_exception(ArgumentError, message: be == "String contains malformed percent encoding!") + end + end + + it "rejects invalid string encoding" do + path = Protocol::URL::Path["/a\xFF".dup.force_encoding(::Encoding::UTF_8)] + + expect do + path.normalize + end.to raise_exception(ArgumentError, message: be == "Path segment has invalid encoding!") + end end with "#simplify!" do @@ -444,6 +532,12 @@ expect(Protocol::URL::Path["documents/report.pdf"].local_path(root)).to be == expected end + it "maps empty URL segments to the same local filesystem path" do + expected = File.join(root, "documents", "report.pdf") + + expect(Protocol::URL::Path["/documents//report.pdf"].local_path(root)).to be == expected + end + it "unescapes percent-encoded and Unicode characters" do expect(Protocol::URL::Path["/files/My%20Document.txt"].local_path(root)).to be == File.join(root, "files", "My Document.txt") expect(Protocol::URL::Path["/files/%E2%9D%A4%EF%B8%8F.txt"].local_path(root)).to be == File.join(root, "files", "❤️.txt") diff --git a/test/protocol/url/relative.rb b/test/protocol/url/relative.rb index 0c652bd..980349c 100644 --- a/test/protocol/url/relative.rb +++ b/test/protocol/url/relative.rb @@ -269,6 +269,18 @@ end with "#normalize!" do + it "normalizes the encoded path" do + url = Protocol::URL::Relative.new("/%66oo/a%2fb") + url.normalize! + expect(url.path).to be == Protocol::URL::Path["/foo/a%2Fb"] + end + + it "simplifies normalized dot segments" do + url = Protocol::URL::Relative.new("/foo/%2e%2e/bar") + url.normalize! + expect(url.path).to be == Protocol::URL::Path["/bar"] + end + it "removes dot segments" do url = Protocol::URL::Relative.new("/foo/./bar") url.normalize! @@ -281,7 +293,7 @@ expect(url.path).to be == Protocol::URL::Path["/foo/baz"] end - it "collapses multiple slashes" do + it "collapses empty path segments" do url = Protocol::URL::Relative.new("/foo//bar///baz") url.normalize! expect(url.path).to be == Protocol::URL::Path["/foo/bar/baz"]