diff --git a/CHANGELOG.md b/CHANGELOG.md index e8cf448..c3a9d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,30 @@ mistaken for a safe patch upgrade. - The module now requires **Go 1.27.2** (was 1.25). Consumers on an older Go toolchain must upgrade, and `go mod tidy` will raise their own `go` directive to at least 1.27.2. +- `marker`, `requirement`, `reqtxt`: **non-ASCII `extra` marker literals now + normalize like Python's `str.lower()`** (see the `extras` entry under + `Fixed`). This changes `String()` output and `Evaluate` results: + `extra == "Ä"` now renders as `extra == "ä"`. `extra == "ΑΣ.Β"` no longer + matches the active extra `ΑΣ_Β` (`ασ-β` vs `ας-β`). Invalid UTF-8 bytes no + longer collapse to U+FFFD, so `"\xff"` no longer matches `"\xfe"`. (#25) ### Added +- `packagename`: new package with `Normalize`, the PEP 503 entry point for + project-name normalization. It returns exactly what `extras.Normalize` + returns; both now delegate to one shared internal implementation. (#25) - `distribution`: Metadata 2.5 (PEP 794) and 2.6 (PEP 808) are now recognized. Metadata 2.5 adds `Import-Name` and `Import-Namespace` fields; 2.6 has no new fields (behavior change for `Dynamic` only). ### Fixed +- `extras`: `Normalize` now lowercases non-ASCII letters the way Python's + `str.lower()` does, matching pypa/packaging's `canonicalize_name`: `Ä` + becomes `ä`, `İ` becomes `i` plus U+0307, and `Σ` becomes `ς` at the end + of a word. Previously only ASCII letters were lowercased. Bytes that are not + valid UTF-8 now pass through unchanged; previously each was replaced with + U+FFFD. Valid PEP 508 names are ASCII, so their results do not change. (#25) - `distribution`: `License-File` is now parsed as a multi-value field, matching the PEP 639 spec. Previously it was incorrectly treated as single-value. diff --git a/CLAUDE.md b/CLAUDE.md index e585754..dde96da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,7 @@ One module, one package per PEP concern (single import path for downstream conve | `requirement/` | PEP 508 dependency specifiers | | `marker/` | PEP 508 environment markers | | `extras/` | PEP 685 extra-name normalization | +| `packagename/` | PEP 503 project-name normalization | | `reqtxt/` | pip `requirements.txt` format | | `distribution/` | METADATA / wheel parsing *(added by migration #18630)* | | `wheelname/` | wheel filename → structured fields | diff --git a/NOTICE b/NOTICE index 0a2482b..98c7743 100644 --- a/NOTICE +++ b/NOTICE @@ -79,10 +79,11 @@ License, Version 2.0; and from Astral's uv (https://github.com/astral-sh/uv), specifically the uv-distribution-filename crate, dual-licensed under the Apache License, Version 2.0, or the MIT license. -The extras/ package adapts PEP 685/503 name-canonicalization from -pypa/packaging (https://github.com/pypa/packaging), specifically +The internal/canonname package adapts PEP 503/685 name-canonicalization +from pypa/packaging (https://github.com/pypa/packaging), specifically packaging/utils.py's `canonicalize_name`, licensed under the Apache -License, Version 2.0. +License, Version 2.0. The extras/ and packagename/ packages expose it as +thin wrappers. The internal/pep508 package's tokenizer adapts the lexer design of pypa/packaging (https://github.com/pypa/packaging), specifically @@ -213,7 +214,8 @@ test_canonicalize_name and the non-validating half of test_canonicalize_name_invalid, pinned at commit 4eb0753dba8fcaaac8eb75463374e448f0931558, licensed under the Apache License, Version 2.0. Translated from Python/pytest to Go table-driven tests, in -extras/conformance_test.go. +extras/conformance_test.go. The test_canonicalize_name cases are ported a +second time, under the same terms, in packagename/conformance_test.go. The internal/pep508 package's error-position conformance tests port input strings and caret span offsets - not message text - from Astral uv's diff --git a/README.md b/README.md index 63939d9..f1e0089 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Part of [RFD 0001 — Native Go PyPI Dependency Resolution](https://github.com/r | `requirement/` | PEP 508 dependency specifiers (implemented) | | `marker/` | PEP 508 environment markers (implemented) | | `extras/` | PEP 685 extra-name normalization (implemented) | +| `packagename/` | PEP 503 project-name normalization (implemented) | | `reqtxt/` | pip `requirements.txt` format (implemented) | | [`distribution/`](https://github.com/rstudio/python-distribution-parser) | METADATA / wheel parsing | | `wheelname/` | PEP 427 wheel filename → name/version/build/tags (implemented) | diff --git a/extras/doc.go b/extras/doc.go index 50b7db6..0c7c1a1 100644 --- a/extras/doc.go +++ b/extras/doc.go @@ -2,8 +2,8 @@ // Package extras normalizes Python extra names per PEP 685. // -// Normalize lowercases an extra name and collapses any run of "-", "_", or -// "." into a single "-", matching the PEP 503 name-canonicalization rule +// Normalize lowercases an extra name as Python's str.lower() does and +// collapses any run of "-", "_", or "." into a single "-", matching the PEP 503 name-canonicalization rule // that PEP 685 mandates for extras. The result is deterministic and // idempotent: normalizing an already-normalized name is a no-op. package extras diff --git a/extras/extras.go b/extras/extras.go index 43436ed..ad24917 100644 --- a/extras/extras.go +++ b/extras/extras.go @@ -1,45 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -// -// Portions of this file are ported from pypa/packaging -// (https://github.com/pypa/packaging), specifically packaging/utils.py's -// canonicalize_name, used under the Apache License, Version 2.0 -// (dual-licensed Apache-2.0 OR BSD-2-Clause; see NOTICE for full license and -// copyright detail). -// Changed: translated from a single re.sub regex into an explicit -// rune-by-rune loop, since extra-name normalization sits on a -// per-dependency hot path; behavior (lowercase, collapse -/_/. runs to one -// "-") is unchanged. package extras -import "strings" +import "github.com/posit-dev/go-python-packaging/internal/canonname" // Normalize canonicalizes a Python extra name per PEP 685: lowercase the // name, then collapse any run of "-", "_", or "." into a single "-". // -// This mirrors pypa/packaging's canonicalize_name (PEP 503/685) without -// using a regular expression, since extra-name normalization sits on a -// per-dependency, per-extra hot path during dependency resolution. +// Lowercasing follows Python's str.lower() across all of Unicode, so the +// result matches pypa/packaging for non-ASCII input too (up to differences +// between Go's and Python's Unicode versions). Bytes that are not valid UTF-8 +// pass through unchanged. +// +// PEP 685 adopts the PEP 503 project-name rule unchanged, so this returns +// exactly what packagename.Normalize returns; both delegate to one shared +// implementation. It mirrors pypa/packaging's canonicalize_name and uses no +// regular expression. func Normalize(name string) string { - var b strings.Builder - b.Grow(len(name)) - - inSep := false - for _, r := range name { - switch r { - case '-', '_', '.': - if !inSep { - b.WriteByte('-') - inSep = true - } - default: - inSep = false - if r >= 'A' && r <= 'Z' { - r += 'a' - 'A' - } - b.WriteRune(r) - } - } - - return b.String() + return canonname.Normalize(name) } diff --git a/internal/canonname/canonname.go b/internal/canonname/canonname.go new file mode 100644 index 0000000..058dabb --- /dev/null +++ b/internal/canonname/canonname.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT +// +// Portions of this file are ported from pypa/packaging +// (https://github.com/pypa/packaging), specifically packaging/utils.py's +// canonicalize_name, used under the Apache License, Version 2.0 +// (dual-licensed Apache-2.0 OR BSD-2-Clause; see NOTICE for full license and +// copyright detail). The Final_Sigma rule in finalSigma follows CPython's +// handle_capital_sigma in Objects/unicodeobject.c (Unicode Standard §3.13). +// Changed: upstream's str.lower(), str.replace, and repeated "--" collapse +// (which emulate PEP 503's re.sub(r"[-_.]+", "-", name).lower()) are +// translated into a single loop with an ASCII fast path and a non-allocating +// early return for already-normalized names, since name normalization sits on +// a per-dependency hot path; behavior (lowercase, collapse -/_/. runs to one +// "-") is unchanged. str.lower() is reproduced with Go's unicode tables, so +// results for characters whose case data differs between Go's and the running +// Python's Unicode version can differ. The validate=True option is not ported. + +// Package canonname holds the single implementation of Python name +// canonicalization shared by PEP 503 project names (package packagename) and +// PEP 685 extra names (package extras). The two PEPs specify the same +// transformation; keeping one implementation means the public entry points +// cannot drift apart. +package canonname + +import ( + "strings" + "unicode" + "unicode/utf8" +) + +const ( + capitalDottedI = 'İ' // U+0130; the only unconditional multi-rune lowercase mapping + capitalSigma = 'Σ' // U+03A3; lowercases to ς or σ depending on context + smallSigma = 'σ' // U+03C3 + finalSmallSig = 'ς' // U+03C2 +) + +// Normalize returns the canonical form of a Python name per PEP 503 (and +// PEP 685, which adopts the same rule): the name is lowercased the way +// Python's str.lower() does it, and any run of "-", "_", or "." collapses to +// a single "-". +// +// Lowercasing covers all of Unicode, not just ASCII, including str.lower()'s +// two special cases: "İ" (U+0130) becomes "i" plus U+0307, and "Σ" becomes +// final "ς" at the end of a word and "σ" elsewhere. Bytes that are not valid +// UTF-8 pass through unchanged. +// +// This mirrors pypa/packaging's canonicalize_name without using a regular +// expression, since it sits on a per-dependency, per-extra hot path during +// dependency resolution. An already-normalized name is returned as-is, +// without allocating. +func Normalize(name string) string { + // Fast path: most names are already normalized; avoid allocating. + if isNormalized(name) { + return name + } + + // strings.Builder, not a []byte plus string(b): String() does not copy, + // so the slow path allocates once. + var b strings.Builder + b.Grow(len(name) + 1) + inSep := false + for i := 0; i < len(name); { + c := name[i] + if c < utf8.RuneSelf { + switch c { + case '-', '_', '.': + if !inSep { + b.WriteByte('-') + inSep = true + } + default: + inSep = false + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + } + b.WriteByte(c) + } + i++ + continue + } + + // No non-ASCII rune lowercases to a separator, so this always ends a + // separator run. + inSep = false + r, size := utf8.DecodeRuneInString(name[i:]) + switch { + case r == utf8.RuneError && size == 1: + b.WriteByte(c) // invalid UTF-8: copy the byte unchanged + case r == capitalDottedI: + b.WriteString("i̇") + case r == capitalSigma: + if finalSigma(name, i, i+size) { + b.WriteRune(finalSmallSig) + } else { + b.WriteRune(smallSigma) + } + default: + b.WriteRune(unicode.ToLower(r)) + } + i += size + } + return b.String() +} + +// isNormalized reports whether Normalize(name) == name: name has no "_" or +// ".", no run of two or more "-", and no rune that lowercasing would change. +func isNormalized(name string) bool { + prevSep := false + for i := 0; i < len(name); { + c := name[i] + if c < utf8.RuneSelf { + switch { + case c == '_' || c == '.', 'A' <= c && c <= 'Z': + return false + case c == '-': + if prevSep { + return false + } + prevSep = true + default: + prevSep = false + } + i++ + continue + } + + prevSep = false + r, size := utf8.DecodeRuneInString(name[i:]) + valid := r != utf8.RuneError || size != 1 + if valid && (r == capitalDottedI || unicode.ToLower(r) != r) { + return false + } + i += size + } + return true +} + +// finalSigma reports whether the capital sigma at name[start:end] is in the +// Final_Sigma context: preceded by a cased letter (skipping case-ignorable +// runes) and not followed by one (again skipping case-ignorable runes). +// +// The context is evaluated on the original name, before separators are +// replaced, because str.lower() runs first upstream. Invalid UTF-8 bytes, +// which Python strings cannot hold, count as neither cased nor +// case-ignorable. +func finalSigma(name string, start, end int) bool { + before := false + for j := start; j > 0; { + r, size := utf8.DecodeLastRuneInString(name[:j]) + if size == 1 && r == utf8.RuneError { + break + } + if !isCaseIgnorable(r) { + before = isCased(r) + break + } + j -= size + } + if !before { + return false + } + for j := end; j < len(name); { + r, size := utf8.DecodeRuneInString(name[j:]) + if size == 1 && r == utf8.RuneError { + return true + } + if !isCaseIgnorable(r) { + return !isCased(r) + } + j += size + } + return true +} + +// isCased reports the Unicode Cased property: Lowercase, Uppercase, or Lt. +func isCased(r rune) bool { + return unicode.IsLower(r) || unicode.IsUpper(r) || unicode.IsTitle(r) || + unicode.Is(unicode.Other_Lowercase, r) || unicode.Is(unicode.Other_Uppercase, r) +} + +// isCaseIgnorable reports the Unicode Case_Ignorable property: general +// category Mn, Me, Cf, Lm, or Sk, or Word_Break MidLetter, MidNumLet, or +// Single_Quote. Go has no Word_Break tables, so those runes are listed. +func isCaseIgnorable(r rune) bool { + switch r { + case '\'', // Single_Quote + '.', '‘', '’', '․', '﹒', '＇', '．', // MidNumLet + ':', '·', '·', '՟', '״', '‧', '︓', '﹕', '：': // MidLetter + return true + } + return unicode.In(r, unicode.Mn, unicode.Me, unicode.Cf, unicode.Lm, unicode.Sk) +} diff --git a/internal/canonname/canonname_test.go b/internal/canonname/canonname_test.go new file mode 100644 index 0000000..8ef1acc --- /dev/null +++ b/internal/canonname/canonname_test.go @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +package canonname + +import ( + "regexp" + "strings" + "testing" + "unicode/utf8" +) + +func TestNormalize(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty string", "", ""}, + {"single character", "x", "x"}, + {"already normal", "already-normal", "already-normal"}, + {"single hyphen", "-", "-"}, + {"uppercase", "Django", "django"}, + {"all uppercase", "PYYAML", "pyyaml"}, + {"mixed case with underscore", "Foo_Bar", "foo-bar"}, + {"dot", "zope.interface", "zope-interface"}, + {"dot and double underscore", "a.b__c", "a-b-c"}, + {"repeated hyphen", "a--b", "a-b"}, + {"uppercase with repeated hyphen", "A---B", "a-b"}, + {"mixed run of separators", "foo_.-bar", "foo-bar"}, + {"uppercase with dots", "Foo.Bar.Baz", "foo-bar-baz"}, + {"leading separators", "__foo", "-foo"}, + {"trailing separators", "foo__", "foo-"}, + {"leading and trailing separators", "..foo..", "-foo-"}, + {"only separators", "___", "-"}, + {"digits unchanged", "Py3_K", "py3-k"}, + {"ASCII bounds around letters", "@AZ[`az{", "@az[`az{"}, + {"whitespace unchanged", "hi\n", "hi\n"}, + {"non-ASCII already normal", "café-au-lait", "café-au-lait"}, + {"multi-byte runes around separators", "日本__語", "日本-語"}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := Normalize(c.in); got != c.want { + t.Errorf("Normalize(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +// Expected values were produced by running pypa/packaging's canonicalize_name +// (pinned 4eb0753dba8fcaaac8eb75463374e448f0931558) under CPython 3.13 +// (Unicode 15.1). +func TestNormalizeMatchesPythonLower(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"Latin-1 uppercase", "ÄBC_def", "äbc-def"}, + {"accented uppercase", "ÀÉÎ", "àéî"}, + {"sharp s unchanged", "Straße", "straße"}, + {"capital sharp s", "ẞ", "ß"}, + {"dotted capital I expands", "İstanbul", "i\u0307stanbul"}, + {"Kelvin sign becomes ASCII k", "\u212Aelvin", "kelvin"}, + {"Angstrom sign", "\u212B", "å"}, + {"titlecase digraph", "Dž", "dž"}, + {"fullwidth", "ＦＵＬＬ", "ｆｕｌｌ"}, + {"Roman numeral", "Ⅻ", "ⅻ"}, + {"Odysseus", "ΟΔΥΣΣΕΥΣ", "οδυσσευς"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := Normalize(c.in); got != c.want { + t.Errorf("Normalize(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +// Σ lowercases to final ς only at the end of a word. Python lowercases +// before replacing separators, so the context is the original name: "_" and +// "-" end a word, but "." and ":" are case-ignorable and do not. Expected +// values were produced the same way as in TestNormalizeMatchesPythonLower. +func TestNormalizeFinalSigma(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"Σ", "σ"}, + {"1Σ", "1σ"}, + {"ΑΣ", "ας"}, + {"ΣΑΣ", "σας"}, + {"ΑΣΒ", "ασβ"}, + {"ΑΣ_Β", "ας-β"}, + {"ΑΣ-Β", "ας-β"}, + {"ΑΣ.Β", "ασ-β"}, + {"ΑΣ:Β", "ασ:β"}, + {"Α.Σ", "α-ς"}, + {"Α_Σ", "α-σ"}, + {"ΑΣ'", "ας'"}, + {"ΑΣ\u0301", "ας\u0301"}, + {"Α\u0301Σ", "α\u0301ς"}, + {"ΑΣ\u0301Β", "ασ\u0301β"}, + {"ΣΑΣ_ΣΑΣ", "σας-σας"}, + } + for _, c := range cases { + if got := Normalize(c.in); got != c.want { + t.Errorf("Normalize(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// Python strings cannot hold invalid UTF-8, so these cases have no upstream +// counterpart; an invalid byte counts as neither cased nor case-ignorable. +func TestNormalizeFinalSigmaInvalidUTF8(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"Α\xffΣ", "α\xffσ"}, + {"ΑΣ\xff", "ας\xff"}, + } + for _, c := range cases { + if got := Normalize(c.in); got != c.want { + t.Errorf("Normalize(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestNormalizeInvalidUTF8PassesThrough(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"\xff", "\xff"}, + {"A\xff_B", "a\xff-b"}, + {"\xc3_", "\xc3-"}, // truncated two-byte sequence followed by a separator + {"\xc3\x84", "ä"}, // the same bytes completed are a valid "Ä" + {"\uFFFD", "\uFFFD"}, // a valid encoded U+FFFD is not mistaken for invalid input + } + for _, c := range cases { + if got := Normalize(c.in); got != c.want { + t.Errorf("Normalize(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestIsNormalized(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"", true}, + {"foo", true}, + {"foo-bar", true}, + {"-", true}, + {"-foo-", true}, + {"a-b-c", true}, + {"café", true}, + {"σας", true}, + {"i̇", true}, + {"\xff", true}, + {"�", true}, + {"Ä", false}, + {"Σ", false}, + {"İ", false}, + {"K", false}, // Kelvin sign lowercases to ASCII "k" + {"Foo", false}, + {"fooZ", false}, + {"foo_bar", false}, + {"foo.bar", false}, + {"--", false}, + {"foo--bar", false}, + {"foo-_bar", false}, + {"_", false}, + {".", false}, + } + for _, c := range cases { + if got := isNormalized(c.in); got != c.want { + t.Errorf("isNormalized(%q) = %v, want %v", c.in, got, c.want) + } + } +} + +func TestNormalizeFastPathDoesNotAllocate(t *testing.T) { + for _, in := range []string{"", "requests", "zope-interface", "café-au-lait"} { + allocs := testing.AllocsPerRun(100, func() { _ = Normalize(in) }) + if allocs != 0 { + t.Errorf("Normalize(%q) allocated %v times, want 0", in, allocs) + } + } +} + +// The input longer than 32 bytes is load-bearing: the compiler stack-allocates +// a small make([]byte, 0, n), so only longer names would expose a second +// allocation from a []byte buffer plus string(b). +func TestNormalizeSlowPathAllocatesOnce(t *testing.T) { + for _, in := range []string{"Django", "zope.interface", "Foo__Bar", "ΑΣ.Β", "İstanbul", "Zope.Interface.Long_Project_Name.Example"} { + allocs := testing.AllocsPerRun(100, func() { _ = Normalize(in) }) + if allocs != 1 { + t.Errorf("Normalize(%q) allocated %v times, want 1", in, allocs) + } + } +} + +// pep503Separators is the separator regex from PEP 503's normalization rule. +var pep503Separators = regexp.MustCompile(`[-_.]+`) + +// reference implements PEP 503's re.sub(r"[-_.]+", "-", name).lower(), the +// form pypa/packaging's canonicalize_name emulates without a regex, with +// strings.ToLower standing in for str.lower(). The two agree except on +// invalid UTF-8 (which strings.ToLower replaces with U+FFFD) and on "İ" and +// "Σ", so ok is false for those inputs; the Python-derived tables above cover +// them. +func reference(name string) (want string, ok bool) { + if !utf8.ValidString(name) || strings.ContainsAny(name, "İΣ") { + return "", false + } + return strings.ToLower(pep503Separators.ReplaceAllString(name, "-")), true +} + +func FuzzNormalize(f *testing.F) { + for _, s := range []string{ + "", "-", "foo", "Foo_Bar", "a.b__c", "A---B", "..foo..", "ÄBC_def", + "日本__語", "\xff", "A\xff_B", "foo-_bar", "ΑΣ.Β", "İstanbul", "Kelvin", + } { + f.Add(s) + } + f.Fuzz(func(t *testing.T, in string) { + got := Normalize(in) + if want, ok := reference(in); ok && got != want { + t.Fatalf("Normalize(%q) = %q, reference = %q", in, got, want) + } + if again := Normalize(got); again != got { + t.Fatalf("not idempotent: Normalize(%q) = %q, Normalize(%q) = %q", in, got, got, again) + } + if isNormalized(in) != (got == in) { + t.Fatalf("isNormalized(%q) = %v, but Normalize(%q) == input is %v", in, isNormalized(in), in, got == in) + } + if !isNormalized(got) { + t.Fatalf("isNormalized(Normalize(%q)) = false for %q", in, got) + } + }) +} diff --git a/marker/evaluate_test.go b/marker/evaluate_test.go index f9f03f2..a01af15 100644 --- a/marker/evaluate_test.go +++ b/marker/evaluate_test.go @@ -178,6 +178,11 @@ func TestEvaluate_Extra_SetMembership(t *testing.T) { {"normalization, literal side", `extra == "Foo_Bar"`, []string{"foo-bar"}, true}, {"normalization, active side", `extra == "foo-bar"`, []string{"Foo_Bar"}, true}, {"normalization, both sides", `extra == "Foo.Bar"`, []string{"foo_bar"}, true}, + // Non-ASCII literals lowercase as Python's str.lower() does: "ΑΣ.Β" + // becomes "ασ-β" but "ΑΣ_Β" becomes "ας-β" (final sigma), so they no + // longer match each other. + {"normalization, non-ASCII", `extra == "Ä"`, []string{"ä"}, true}, + {"normalization, final sigma", `extra == "ΑΣ.Β"`, []string{"ΑΣ_Β"}, false}, // Only == and != are meaningful for extra set-membership; any other // operator is inapplicable and evaluates false, even when the extra // named on the value side is active. diff --git a/marker/marker_test.go b/marker/marker_test.go index 0de7872..640d7c6 100644 --- a/marker/marker_test.go +++ b/marker/marker_test.go @@ -59,6 +59,14 @@ func TestParse_ExtraNormalization(t *testing.T) { assert.Equal(t, `extra == "foo-bar"`, m.String()) } +// A marker `extra` literal is an arbitrary quoted string, so non-ASCII text +// reaches extras.Normalize and is lowercased the way Python's str.lower() does. +func TestParse_ExtraNormalization_NonASCII(t *testing.T) { + m, err := Parse(`extra == "Ä"`) + require.NoError(t, err) + assert.Equal(t, `extra == "ä"`, m.String()) +} + // --- round trip: Parse(m.String()) is stable --- func TestParse_RoundTrip(t *testing.T) { diff --git a/packagename/conformance_test.go b/packagename/conformance_test.go new file mode 100644 index 0000000..517ea18 --- /dev/null +++ b/packagename/conformance_test.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT +// +// Portions of this file port test cases from pypa/packaging +// (https://github.com/pypa/packaging), specifically tests/test_utils.py's +// test_canonicalize_name, used under the Apache License, Version 2.0 +// (dual-licensed Apache-2.0 OR BSD-2-Clause; see NOTICE for full license and +// copyright detail). +// Changed: translated from Python/pytest to a Go table-driven test. +// +// These are the same cases extras/conformance_test.go ports. They are repeated +// here under the PEP 503 project-name framing that canonicalize_name was +// written for, so this entry point is pinned to upstream on its own. +// +// Upstream pinned at 4eb0753dba8fcaaac8eb75463374e448f0931558. + +package packagename + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ported from pypa/packaging tests/test_utils.py, test_canonicalize_name. +// Pinned: 4eb0753dba8fcaaac8eb75463374e448f0931558 (L23-L39 at that commit). +// https://github.com/pypa/packaging/blob/4eb0753dba8fcaaac8eb75463374e448f0931558/tests/test_utils.py#L23-L39 +func TestConformance_CanonicalizeName(t *testing.T) { + tests := []struct { + name string + want string + }{ + {"foo", "foo"}, + {"Foo", "foo"}, + {"fOo", "foo"}, + {"foo.bar", "foo-bar"}, + {"Foo.Bar", "foo-bar"}, + {"Foo.....Bar", "foo-bar"}, + {"foo_bar", "foo-bar"}, + {"foo___bar", "foo-bar"}, + {"foo-bar", "foo-bar"}, + {"foo----bar", "foo-bar"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, Normalize(tt.name)) + }) + } +} diff --git a/packagename/doc.go b/packagename/doc.go new file mode 100644 index 0000000..4ae5191 --- /dev/null +++ b/packagename/doc.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// Package packagename normalizes Python project (distribution) names per +// PEP 503. +// +// Normalize lowercases a project name as Python's str.lower() does and +// collapses any run of "-", "_", or "." into a single "-". +// The result is deterministic and idempotent: normalizing an +// already-normalized name is a no-op. +// +// PEP 685 extra names use the same rule, so packagename.Normalize and +// extras.Normalize always agree; both delegate to one shared implementation. +// Call the one whose name matches what you are normalizing. +package packagename diff --git a/packagename/packagename.go b/packagename/packagename.go new file mode 100644 index 0000000..3156f24 --- /dev/null +++ b/packagename/packagename.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +package packagename + +import "github.com/posit-dev/go-python-packaging/internal/canonname" + +// Normalize canonicalizes a Python project name per PEP 503: lowercase the +// name, then collapse any run of "-", "_", or "." into a single "-". +// +// Lowercasing follows Python's str.lower() across all of Unicode, so the +// result matches pypa/packaging for non-ASCII input too (up to differences +// between Go's and Python's Unicode versions). Bytes that are not valid UTF-8 +// pass through unchanged. +// +// It mirrors pypa/packaging's canonicalize_name (without validate=True) and +// uses no regular expression. It does not check that name is a valid project +// name. +func Normalize(name string) string { + return canonname.Normalize(name) +} diff --git a/packagename/packagename_test.go b/packagename/packagename_test.go new file mode 100644 index 0000000..7c4e160 --- /dev/null +++ b/packagename/packagename_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +package packagename + +import ( + "testing" + + "github.com/posit-dev/go-python-packaging/extras" +) + +// TestNormalizeMatchesExtras checks that the PEP 503 and PEP 685 entry points +// agree, including on non-ASCII and invalid names. The transformation itself is covered by +// extras/extras_test.go and both conformance tables. +func TestNormalizeMatchesExtras(t *testing.T) { + inputs := []string{ + "", "x", "Django", "zope.interface", "Flask_SQLAlchemy", + "ruamel.yaml.clib", "A---B", "__foo", "foo__", "..foo..", "___", + "foo_.-bar", "ÄBC_def", "hi\n", + } + for _, in := range inputs { + if got, want := Normalize(in), extras.Normalize(in); got != want { + t.Errorf("packagename.Normalize(%q) = %q, extras.Normalize(%q) = %q", in, got, in, want) + } + } +}