Add SWAR versions of Base validations#15357
Merged
Merged
Conversation
Contributor
Author
Benchmark# SWAR exploration bench for Base.valid* functions.
#
# Run with the SYSTEM elixir from anywhere:
# elixir bench_base_swar.ex # all benchmarks
# elixir bench_base_swar.ex valid64 # just valid64?
# elixir bench_base_swar.ex valid64 valid32 # both
#
# Recognised names: valid16, valid32, hex_valid32, valid64, url_valid64.
# Equivalence checks always run regardless of selection (they're fast).
#
# Self-contained: the BASELINE is a copy of the current valid* logic from
# lib/elixir/lib/base.ex, inlined into a `Baseline` module so the bench does
# not depend on `make stdlib`. The SWAR variants live in `Optimised`, in the
# same style as PR #15255.
Mix.install([{:benchee, "~> 1.5"}])
selected = System.argv()
run? = fn name -> selected == [] or name in selected end
defmodule Baseline do
@moduledoc false
# Mirrors the b16 setup in lib/elixir/lib/base.ex.
b16_alphabet = ~c"0123456789ABCDEF"
to_lower_dec =
&Enum.map(&1, fn {encoding, value} = pair ->
if encoding in ?A..?Z do
{encoding - ?A + ?a, value}
else
pair
end
end)
to_mixed_dec =
&Enum.flat_map(&1, fn {encoding, value} = pair ->
if encoding in ?A..?Z do
[pair, {encoding - ?A + ?a, value}]
else
[pair]
end
end)
to_decode_list = fn alphabet ->
alphabet = Enum.sort(alphabet)
map = Map.new(alphabet)
{min, _} = List.first(alphabet)
{max, _} = List.last(alphabet)
{min, Enum.map(min..max, &map[&1])}
end
def valid16?(string, opts \\ [])
def valid16?(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do
case Keyword.get(opts, :case, :upper) do
:upper -> validate16upper?(string)
:lower -> validate16lower?(string)
:mixed -> validate16mixed?(string)
end
end
def valid16?(string, _opts) when is_binary(string) do
false
end
upper = Enum.with_index(b16_alphabet)
for {base, alphabet} <- [upper: upper, lower: to_lower_dec.(upper), mixed: to_mixed_dec.(upper)] do
validate_name = :"validate16#{base}?"
valid_char_name = :"valid_char16#{base}?"
{min, decoded} = to_decode_list.(alphabet)
defp unquote(validate_name)(<<>>), do: true
defp unquote(validate_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<c1, c2, c3, c4, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<c1, c2, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<_char, _rest::binary>>), do: false
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
# --- base32 (mirrors lib/elixir/lib/base.ex valid32? machinery) ----------
b32_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
b32hex_alphabet = ~c"0123456789ABCDEFGHIJKLMNOPQRSTUV"
upper32 = Enum.with_index(b32_alphabet)
hexupper32 = Enum.with_index(b32hex_alphabet)
def valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32upper?(string, pad?)
:lower -> validate32lower?(string, pad?)
:mixed -> validate32mixed?(string, pad?)
end
end
def hex_valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32hexupper?(string, pad?)
:lower -> validate32hexlower?(string, pad?)
:mixed -> validate32hexmixed?(string, pad?)
end
end
for {base, alphabet} <- [
upper: upper32,
lower: to_lower_dec.(upper32),
mixed: to_mixed_dec.(upper32),
hexupper: hexupper32,
hexlower: to_lower_dec.(hexupper32),
hexmixed: to_mixed_dec.(hexupper32)
] do
validate_name = :"validate32#{base}?"
validate_main_name = :"validate_main32#{base}?"
valid_char_name = :"valid_char32#{base}?"
{min, decoded} = to_decode_list.(alphabet)
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_main_name)(rest)
end
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=, ?=, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, c4, ?=, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and unquote(valid_char_name)(c8)
<<c1, c2>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, c4>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
_ ->
false
end
end
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
# --- base64 (mirrors lib/elixir/lib/base.ex valid64?/url_valid64? machinery)
b64_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
b64url_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
defp remove_ignored(string, nil), do: string
defp remove_ignored(string, :whitespace) do
for <<char::8 <- string>>, char not in ~c"\s\t\r\n", into: <<>>, do: <<char::8>>
end
def valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64base?(pad?)
end
def url_valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64url?(pad?)
end
for {base, alphabet} <- [base: b64_alphabet, url: b64url_alphabet] do
validate_name = :"validate64#{base}?"
validate_main_name = :"validate_main64#{base}?"
valid_char_name = :"valid_char64#{base}?"
{min, decoded} = alphabet |> Enum.with_index() |> to_decode_list.()
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_main_name)(rest)
end
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3)
<<c1, c2, c3, c4>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5, c6, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and unquote(valid_char_name)(c8)
<<c1, c2>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3)
<<c1, c2, c3, c4, c5, c6>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
_ ->
false
end
end
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
end
defmodule Optimised do
@moduledoc false
import Bitwise
# SWAR-optimised valid16?/2 and valid32?/2 for :upper, :lower and :mixed.
#
# 56 bits = largest integer that fits in a BEAM small int (fixnum range is
# 59-bit signed on 64-bit OTP). At 64 bits, every `w + 0x80..` would
# allocate a bignum on the heap and the optimisation collapses.
# See https://github.com/erlang/otp/pull/10938.
@swar_mask80 0x80808080808080
# Per-range SWAR constants, broadcast across 7 lanes. Naming convention:
# @swar_ge_X = 0x80 - X → (w + @swar_ge_X) has high bit set iff byte ≥ X
# @swar_gt_X = 0x7F - X → (w + @swar_gt_X) has high bit set iff byte > X
# A byte is in range [lo, hi] iff (bxor(w + @swar_ge_lo, w + @swar_gt_hi))
# has its high bit set.
@swar_ge_0 0x50505050505050
@swar_gt_9 0x46464646464646
@swar_ge_2 0x4E4E4E4E4E4E4E
@swar_gt_7 0x48484848484848
@swar_ge_A 0x3F3F3F3F3F3F3F
@swar_gt_F 0x39393939393939
@swar_gt_V 0x29292929292929
@swar_gt_Z 0x25252525252525
@swar_ge_a 0x1F1F1F1F1F1F1F
@swar_gt_f 0x19191919191919
@swar_gt_v 0x09090909090909
@swar_gt_z 0x05050505050505
# Mycroft zero-byte detection for base64 singletons (+, -, _).
# Per lane: high bit set iff `bxor(w, K*ones) - 0x01..01` has its high bit
# set, i.e. that byte's V value was 0 → original byte was K. Simplified
# (no `bnot V` term) — for ASCII-gated `w`, borrow propagation false
# positives only occur for adjacent bytes that happen to equal `K xor 0x01`,
# which is outside the base64 alphabet, so it never matters here.
# Pattern follows https://github.com/elixir-lang/elixir/pull/15255.
@swar_mask01 0x01010101010101
@swar_plus_x7 0x2B2B2B2B2B2B2B
@swar_dash_x7 0x2D2D2D2D2D2D2D
@swar_under_x7 0x5F5F5F5F5F5F5F
# For base64 standard, '/' (0x2F) sits exactly one below '0' (0x30), so we
# extend the digit range to [0x2F, 0x39] which catches '/' as part of one
# range check — saves one Mycroft singleton. Trick lifted from
# https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/
@swar_ge_slash 0x51515151515151
# Per-byte validity. One guard per (encoding, case). Used in the SWAR
# clauses for the 8th byte of the stride, and in the body of tail clauses.
defguardp valid_char16upper?(c) when c in ?0..?9 or c in ?A..?F
defguardp valid_char16lower?(c) when c in ?0..?9 or c in ?a..?f
defguardp valid_char16mixed?(c) when c in ?0..?9 or c in ?A..?F or c in ?a..?f
defguardp valid_char32upper?(c) when c in ?A..?Z or c in ?2..?7
defguardp valid_char32lower?(c) when c in ?a..?z or c in ?2..?7
defguardp valid_char32mixed?(c) when c in ?A..?Z or c in ?a..?z or c in ?2..?7
# Most common range first — for short-circuit OR, this minimises avg
# comparisons in the 8th-byte SWAR check + per-byte tail. In hex base32,
# letters dominate (22/32) over digits (10/32), so letters go first.
defguardp valid_char32hexupper?(c) when c in ?A..?V or c in ?0..?9
defguardp valid_char32hexlower?(c) when c in ?a..?v or c in ?0..?9
defguardp valid_char32hexmixed?(c) when c in ?A..?V or c in ?a..?v or c in ?0..?9
# base64 alphabets have 3 ranges (A-Z, a-z, 0-9) + 2 singletons. Singletons
# are excluded from SWAR; chunks containing them fall to per-byte. Order:
# letters most common (~82%), digits (~16%), singletons (~3%).
defguardp valid_char64base?(c)
when c in ?A..?Z or c in ?a..?z or c in ?0..?9 or c == ?+ or c == ?/
defguardp valid_char64url?(c)
when c in ?A..?Z or c in ?a..?z or c in ?0..?9 or c == ?- or c == ?_
# SWAR 7-byte word validity. One guard per (encoding, case).
#
# Structure:
# 1. ASCII gate `band(w, MASK80) == 0`:
# every byte < 0x80 so the additions below cannot carry across lanes.
#
# 2. "Each byte is in range A OR range B (OR range C)" gate:
# For each valid range [lo, hi], one SWAR mask
# (w + @swar_ge_lo) bxor (w + @swar_gt_hi)
# has high bit = 1 in lanes where the byte is in [lo, hi]. OR the
# per-range masks, AND with MASK80, demand all 7 high bits are set.
defguardp valid_word16upper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_F)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word16lower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_a, w + @swar_gt_f)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word16mixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_F)
),
bxor(w + @swar_ge_a, w + @swar_gt_f)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32upper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32lower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_a, w + @swar_gt_z),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32mixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexupper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_V)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexlower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_a, w + @swar_gt_v)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexmixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_V)
),
bxor(w + @swar_ge_a, w + @swar_gt_v)
),
@swar_mask80
) == @swar_mask80
# SWAR for base64 standard: 3 ranges OR'd with a single Mycroft singleton
# for '+'. The digit range is extended to [0x2F, 0x39] so it absorbs '/'
# (0x2F) — Lemire-style range/singleton merge.
defguardp valid_word64base?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_slash, w + @swar_gt_9)
),
bxor(w, @swar_plus_x7) - @swar_mask01
),
@swar_mask80
) == @swar_mask80
defguardp valid_word64url?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_0, w + @swar_gt_9)
),
bor(
bxor(w, @swar_dash_x7) - @swar_mask01,
bxor(w, @swar_under_x7) - @swar_mask01
)
),
@swar_mask80
) == @swar_mask80
# =========================================================================
# base16
# =========================================================================
def valid16?(string, opts \\ [])
def valid16?(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do
case Keyword.get(opts, :case, :upper) do
:upper -> validate16upper?(string)
:lower -> validate16lower?(string)
:mixed -> validate16mixed?(string)
end
end
def valid16?(string, _opts) when is_binary(string), do: false
# --- :upper -------------------------------------------------------------
defp validate16upper?(<<w::56, b, rest::binary>>)
when valid_word16upper?(w) and valid_char16upper?(b),
do: validate16upper?(rest)
defp validate16upper?(<<_::56, _, _::binary>>), do: false
defp validate16upper?(<<c1, c2, c3, c4, rest::binary>>) do
valid_char16upper?(c1) and
valid_char16upper?(c2) and
valid_char16upper?(c3) and
valid_char16upper?(c4) and
validate16upper?(rest)
end
defp validate16upper?(<<c1, c2, rest::binary>>) do
valid_char16upper?(c1) and
valid_char16upper?(c2) and
validate16upper?(rest)
end
defp validate16upper?(<<>>), do: true
defp validate16upper?(_), do: false
# --- :lower -------------------------------------------------------------
defp validate16lower?(<<w::56, b, rest::binary>>)
when valid_word16lower?(w) and valid_char16lower?(b),
do: validate16lower?(rest)
defp validate16lower?(<<_::56, _, _::binary>>), do: false
defp validate16lower?(<<c1, c2, c3, c4, rest::binary>>) do
valid_char16lower?(c1) and
valid_char16lower?(c2) and
valid_char16lower?(c3) and
valid_char16lower?(c4) and
validate16lower?(rest)
end
defp validate16lower?(<<c1, c2, rest::binary>>) do
valid_char16lower?(c1) and
valid_char16lower?(c2) and
validate16lower?(rest)
end
defp validate16lower?(<<>>), do: true
defp validate16lower?(_), do: false
# --- :mixed -------------------------------------------------------------
defp validate16mixed?(<<w::56, b, rest::binary>>)
when valid_word16mixed?(w) and valid_char16mixed?(b),
do: validate16mixed?(rest)
defp validate16mixed?(<<_::56, _, _::binary>>), do: false
defp validate16mixed?(<<c1, c2, c3, c4, rest::binary>>) do
valid_char16mixed?(c1) and
valid_char16mixed?(c2) and
valid_char16mixed?(c3) and
valid_char16mixed?(c4) and
validate16mixed?(rest)
end
defp validate16mixed?(<<c1, c2, rest::binary>>) do
valid_char16mixed?(c1) and
valid_char16mixed?(c2) and
validate16mixed?(rest)
end
defp validate16mixed?(<<>>), do: true
defp validate16mixed?(_), do: false
# =========================================================================
# base32
# =========================================================================
def valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32upper?(string, pad?)
:lower -> validate32lower?(string, pad?)
:mixed -> validate32mixed?(string, pad?)
end
end
def hex_valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32hexupper?(string, pad?)
:lower -> validate32hexlower?(string, pad?)
:mixed -> validate32hexmixed?(string, pad?)
end
end
# The base32 stride mirrors Baseline: split into `main` (multiple of 8 bytes)
# and a `rest` (1-8 bytes) so the last block's padding patterns are handled
# per-byte. SWAR only fast-paths the `main` loop. Same machinery covers
# both the regular alphabet (`:upper`/`:lower`/`:mixed`) and the
# extended-hex alphabet (`:hexupper`/`:hexlower`/`:hexmixed`).
for {case_name, char_guard, word_guard} <- [
{:upper, :valid_char32upper?, :valid_word32upper?},
{:lower, :valid_char32lower?, :valid_word32lower?},
{:mixed, :valid_char32mixed?, :valid_word32mixed?},
{:hexupper, :valid_char32hexupper?, :valid_word32hexupper?},
{:hexlower, :valid_char32hexlower?, :valid_word32hexlower?},
{:hexmixed, :valid_char32hexmixed?, :valid_word32hexmixed?}
] do
validate_name = :"validate32#{case_name}?"
validate_main_name = :"validate_main32#{case_name}?"
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=, ?=, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, c4, ?=, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7) and unquote(char_guard)(c8)
<<c1, c2>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, c4>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
_ ->
false
end
end
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<w::56, b, rest::binary>>)
when unquote(word_guard)(w) and unquote(char_guard)(b),
do: unquote(validate_main_name)(rest)
defp unquote(validate_main_name)(<<_::56, _, _::binary>>), do: false
end
# =========================================================================
# base64
# =========================================================================
defp remove_ignored(string, nil), do: string
defp remove_ignored(string, :whitespace) do
for <<char::8 <- string>>, char not in ~c"\s\t\r\n", into: <<>>, do: <<char::8>>
end
def valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64base?(pad?)
end
def url_valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64url?(pad?)
end
# Same dispatch shape as base32: split into `main` (multiple of 8 bytes) and
# `rest` (≤8 bytes containing padding). SWAR includes singletons via
# Mycroft, so no per-byte fallback is needed in the main loop.
for {variant, char_guard, word_guard} <- [
{:base, :valid_char64base?, :valid_word64base?},
{:url, :valid_char64url?, :valid_word64url?}
] do
validate_name = :"validate64#{variant}?"
validate_main_name = :"validate_main64#{variant}?"
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and unquote(char_guard)(c3)
<<c1, c2, c3, c4>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5, c6, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7) and unquote(char_guard)(c8)
<<c1, c2>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and unquote(char_guard)(c3)
<<c1, c2, c3, c4, c5, c6>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
_ ->
false
end
end
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<w::56, b, rest::binary>>)
when unquote(word_guard)(w) and unquote(char_guard)(b),
do: unquote(validate_main_name)(rest)
defp unquote(validate_main_name)(<<_::56, _, _::binary>>), do: false
end
end
sizes = [
{"1KiB", 1 * 1024},
{"100KiB", 100 * 1024},
{"1MiB", 1024 * 1024}
]
# Each input is `{binary, case_opt}` so both scenarios (Baseline / Optimised)
# call the same function with the same arguments and Benchee shows them
# head-to-head per input row.
inputs =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.encode16(data),
lower = Base.encode16(data, case: :lower),
# Mixed-case: flip every other letter to lowercase.
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?F and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path shapes (upper only; 'Z' is invalid in all three modes anyway).
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "Z",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"Z" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
# Sanity-check Baseline and Optimised against the stdlib Base for every input.
for {label, {bin, case_opt}} <- inputs do
ref = Base.valid16?(bin, case: case_opt)
b = Baseline.valid16?(bin, case: case_opt)
o = Optimised.valid16?(bin, case: case_opt)
ref === b ||
raise "Baseline mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid16? equivalence check passed.\n")
# --- base32 inputs ------------------------------------------------------
inputs32 =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.encode32(data),
lower = Base.encode32(data, case: :lower),
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?Z and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path: '!' (0x21) is invalid in every base32 alphabet.
# Replacing the last char may overwrite a `=` pad byte; that's fine —
# both impls then fall through their `case rest do` to `_ -> false`.
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "!",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"!" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
for {label, {bin, case_opt}} <- inputs32 do
ref = Base.valid32?(bin, case: case_opt)
b = Baseline.valid32?(bin, case: case_opt)
o = Optimised.valid32?(bin, case: case_opt)
ref === b ||
raise "Baseline valid32? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised valid32? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid32? equivalence check passed.\n")
# --- base32 hex inputs --------------------------------------------------
inputs32hex =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.hex_encode32(data),
lower = Base.hex_encode32(data, case: :lower),
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?V and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path: '!' (0x21) is invalid in every base32hex alphabet too.
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "!",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"!" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
for {label, {bin, case_opt}} <- inputs32hex do
ref = Base.hex_valid32?(bin, case: case_opt)
b = Baseline.hex_valid32?(bin, case: case_opt)
o = Optimised.hex_valid32?(bin, case: case_opt)
ref === b ||
raise "Baseline hex_valid32? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised hex_valid32? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# hex_valid32? equivalence check passed.\n")
# --- base64 inputs ------------------------------------------------------
inputs64base =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
base = Base.encode64(data),
# Slow-path: '!' (0x21) is invalid in every base64 alphabet.
invalid_at_end = binary_part(base, 0, byte_size(base) - 1) <> "!",
mid = div(byte_size(base), 2),
invalid_at_mid =
binary_part(base, 0, mid) <>
"!" <> binary_part(base, mid + 1, byte_size(base) - mid - 1),
{label, payload} <- [
{"#{size_label} valid", base},
{"#{size_label} invalid@end", invalid_at_end},
{"#{size_label} invalid@mid", invalid_at_mid}
],
into: %{},
do: {label, payload}
inputs64url =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
url = Base.url_encode64(data),
invalid_at_end = binary_part(url, 0, byte_size(url) - 1) <> "!",
mid = div(byte_size(url), 2),
invalid_at_mid =
binary_part(url, 0, mid) <>
"!" <> binary_part(url, mid + 1, byte_size(url) - mid - 1),
{label, payload} <- [
{"#{size_label} valid", url},
{"#{size_label} invalid@end", invalid_at_end},
{"#{size_label} invalid@mid", invalid_at_mid}
],
into: %{},
do: {label, payload}
for {label, bin} <- inputs64base do
ref = Base.valid64?(bin)
b = Baseline.valid64?(bin)
o = Optimised.valid64?(bin)
ref === b ||
raise "Baseline valid64? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised valid64? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
for {label, bin} <- inputs64url do
ref = Base.url_valid64?(bin)
b = Baseline.url_valid64?(bin)
o = Optimised.url_valid64?(bin)
ref === b ||
raise "Baseline url_valid64? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised url_valid64? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid64? / url_valid64? equivalence check passed.\n")
if run?.("valid16") do
IO.puts("\n========== valid16? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.valid16?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.valid16?(s, case: c) end
},
inputs: inputs,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("valid32") do
IO.puts("\n========== valid32? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.valid32?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.valid32?(s, case: c) end
},
inputs: inputs32,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("hex_valid32") do
IO.puts("\n========== hex_valid32? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.hex_valid32?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.hex_valid32?(s, case: c) end
},
inputs: inputs32hex,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("valid64") do
IO.puts("\n========== valid64? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn s -> Baseline.valid64?(s) end,
"Optimised" => fn s -> Optimised.valid64?(s) end
},
inputs: inputs64base,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("url_valid64") do
IO.puts("\n========== url_valid64? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn s -> Baseline.url_valid64?(s) end,
"Optimised" => fn s -> Optimised.url_valid64?(s) end
},
inputs: inputs64url,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end |
Contributor
Author
Results |
josevalim
reviewed
May 11, 2026
Member
The results for Even if the results are not so great, I'd apply it to the hex ones for consistency and to reduce code branches! |
Contributor
Author
|
@josevalim you sir are a genius! I removed the guard check on the 8th character and run the 7 bytes through SWAR now. That improved performance from e.g. 1.25x to 2.5x! New Results |
Contributor
Author
Updated Benchmark# SWAR exploration bench for Base.valid* functions.
#
# Run with the SYSTEM elixir from anywhere:
# elixir bench_base_swar.ex # all benchmarks
# elixir bench_base_swar.ex valid64 # just valid64?
# elixir bench_base_swar.ex valid64 valid32 # both
#
# Recognised names: valid16, valid32, hex_valid32, valid64, url_valid64.
# Equivalence checks always run regardless of selection (they're fast).
#
# Self-contained: the BASELINE is a copy of the current valid* logic from
# lib/elixir/lib/base.ex, inlined into a `Baseline` module so the bench does
# not depend on `make stdlib`. The SWAR variants live in `Optimised`, in the
# same style as PR #15255.
Mix.install([{:benchee, "~> 1.5"}])
selected = System.argv()
run? = fn name -> selected == [] or name in selected end
defmodule Baseline do
@moduledoc false
# Mirrors the b16 setup in lib/elixir/lib/base.ex.
b16_alphabet = ~c"0123456789ABCDEF"
to_lower_dec =
&Enum.map(&1, fn {encoding, value} = pair ->
if encoding in ?A..?Z do
{encoding - ?A + ?a, value}
else
pair
end
end)
to_mixed_dec =
&Enum.flat_map(&1, fn {encoding, value} = pair ->
if encoding in ?A..?Z do
[pair, {encoding - ?A + ?a, value}]
else
[pair]
end
end)
to_decode_list = fn alphabet ->
alphabet = Enum.sort(alphabet)
map = Map.new(alphabet)
{min, _} = List.first(alphabet)
{max, _} = List.last(alphabet)
{min, Enum.map(min..max, &map[&1])}
end
def valid16?(string, opts \\ [])
def valid16?(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do
case Keyword.get(opts, :case, :upper) do
:upper -> validate16upper?(string)
:lower -> validate16lower?(string)
:mixed -> validate16mixed?(string)
end
end
def valid16?(string, _opts) when is_binary(string) do
false
end
upper = Enum.with_index(b16_alphabet)
for {base, alphabet} <- [upper: upper, lower: to_lower_dec.(upper), mixed: to_mixed_dec.(upper)] do
validate_name = :"validate16#{base}?"
valid_char_name = :"valid_char16#{base}?"
{min, decoded} = to_decode_list.(alphabet)
defp unquote(validate_name)(<<>>), do: true
defp unquote(validate_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<c1, c2, c3, c4, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<c1, c2, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(validate_name)(rest)
end
defp unquote(validate_name)(<<_char, _rest::binary>>), do: false
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
# --- base32 (mirrors lib/elixir/lib/base.ex valid32? machinery) ----------
b32_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
b32hex_alphabet = ~c"0123456789ABCDEFGHIJKLMNOPQRSTUV"
upper32 = Enum.with_index(b32_alphabet)
hexupper32 = Enum.with_index(b32hex_alphabet)
def valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32upper?(string, pad?)
:lower -> validate32lower?(string, pad?)
:mixed -> validate32mixed?(string, pad?)
end
end
def hex_valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32hexupper?(string, pad?)
:lower -> validate32hexlower?(string, pad?)
:mixed -> validate32hexmixed?(string, pad?)
end
end
for {base, alphabet} <- [
upper: upper32,
lower: to_lower_dec.(upper32),
mixed: to_mixed_dec.(upper32),
hexupper: hexupper32,
hexlower: to_lower_dec.(hexupper32),
hexmixed: to_mixed_dec.(hexupper32)
] do
validate_name = :"validate32#{base}?"
validate_main_name = :"validate_main32#{base}?"
valid_char_name = :"valid_char32#{base}?"
{min, decoded} = to_decode_list.(alphabet)
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_main_name)(rest)
end
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=, ?=, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, c4, ?=, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5, ?=, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and unquote(valid_char_name)(c8)
<<c1, c2>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, c4>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
_ ->
false
end
end
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
# --- base64 (mirrors lib/elixir/lib/base.ex valid64?/url_valid64? machinery)
b64_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
b64url_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
defp remove_ignored(string, nil), do: string
defp remove_ignored(string, :whitespace) do
for <<char::8 <- string>>, char not in ~c"\s\t\r\n", into: <<>>, do: <<char::8>>
end
def valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64base?(pad?)
end
def url_valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64url?(pad?)
end
for {base, alphabet} <- [base: b64_alphabet, url: b64url_alphabet] do
validate_name = :"validate64#{base}?"
validate_main_name = :"validate_main64#{base}?"
valid_char_name = :"valid_char64#{base}?"
{min, decoded} = alphabet |> Enum.with_index() |> to_decode_list.()
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<c1, c2, c3, c4, c5, c6, c7, c8, rest::binary>>) do
unquote(valid_char_name)(c1) and
unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and
unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and
unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and
unquote(valid_char_name)(c8) and
unquote(validate_main_name)(rest)
end
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3)
<<c1, c2, c3, c4>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4)
<<c1, c2, c3, c4, c5, c6, ?=, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7) and unquote(valid_char_name)(c8)
<<c1, c2>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2)
<<c1, c2, c3>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3)
<<c1, c2, c3, c4, c5, c6>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(valid_char_name)(c1) and unquote(valid_char_name)(c2) and
unquote(valid_char_name)(c3) and unquote(valid_char_name)(c4) and
unquote(valid_char_name)(c5) and unquote(valid_char_name)(c6) and
unquote(valid_char_name)(c7)
_ ->
false
end
end
@compile {:inline, [{valid_char_name, 1}]}
defp unquote(valid_char_name)(char)
when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil,
do: true
defp unquote(valid_char_name)(_char), do: false
end
end
defmodule Optimised do
@moduledoc false
import Bitwise
# SWAR-optimised valid16?/2 and valid32?/2 for :upper, :lower and :mixed.
#
# 56 bits = largest integer that fits in a BEAM small int (fixnum range is
# 59-bit signed on 64-bit OTP). At 64 bits, every `w + 0x80..` would
# allocate a bignum on the heap and the optimisation collapses.
# See https://github.com/erlang/otp/pull/10938.
@swar_mask80 0x80808080808080
# Per-range SWAR constants, broadcast across 7 lanes. Naming convention:
# @swar_ge_X = 0x80 - X → (w + @swar_ge_X) has high bit set iff byte ≥ X
# @swar_gt_X = 0x7F - X → (w + @swar_gt_X) has high bit set iff byte > X
# A byte is in range [lo, hi] iff (bxor(w + @swar_ge_lo, w + @swar_gt_hi))
# has its high bit set.
@swar_ge_0 0x50505050505050
@swar_gt_9 0x46464646464646
@swar_ge_2 0x4E4E4E4E4E4E4E
@swar_gt_7 0x48484848484848
@swar_ge_A 0x3F3F3F3F3F3F3F
@swar_gt_F 0x39393939393939
@swar_gt_V 0x29292929292929
@swar_gt_Z 0x25252525252525
@swar_ge_a 0x1F1F1F1F1F1F1F
@swar_gt_f 0x19191919191919
@swar_gt_v 0x09090909090909
@swar_gt_z 0x05050505050505
# Mycroft zero-byte detection for base64 singletons (+, -, _).
# Per lane: high bit set iff `bxor(w, K*ones) - 0x01..01` has its high bit
# set, i.e. that byte's V value was 0 → original byte was K. Simplified
# (no `bnot V` term) — for ASCII-gated `w`, borrow propagation false
# positives only occur for adjacent bytes that happen to equal `K xor 0x01`,
# which is outside the base64 alphabet, so it never matters here.
# Pattern follows https://github.com/elixir-lang/elixir/pull/15255.
@swar_mask01 0x01010101010101
@swar_plus_x7 0x2B2B2B2B2B2B2B
@swar_dash_x7 0x2D2D2D2D2D2D2D
@swar_under_x7 0x5F5F5F5F5F5F5F
# For base64 standard, '/' (0x2F) sits exactly one below '0' (0x30), so we
# extend the digit range to [0x2F, 0x39] which catches '/' as part of one
# range check — saves one Mycroft singleton. Trick lifted from
# https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/
@swar_ge_slash 0x51515151515151
# Per-byte validity. One guard per (encoding, case). Used in the SWAR
# clauses for the 8th byte of the stride, and in the body of tail clauses.
defguardp valid_char16upper?(c) when c in ?0..?9 or c in ?A..?F
defguardp valid_char16lower?(c) when c in ?0..?9 or c in ?a..?f
defguardp valid_char16mixed?(c) when c in ?0..?9 or c in ?A..?F or c in ?a..?f
defguardp valid_char32upper?(c) when c in ?A..?Z or c in ?2..?7
defguardp valid_char32lower?(c) when c in ?a..?z or c in ?2..?7
defguardp valid_char32mixed?(c) when c in ?A..?Z or c in ?a..?z or c in ?2..?7
# Most common range first — for short-circuit OR, this minimises avg
# comparisons in the 8th-byte SWAR check + per-byte tail. In hex base32,
# letters dominate (22/32) over digits (10/32), so letters go first.
defguardp valid_char32hexupper?(c) when c in ?A..?V or c in ?0..?9
defguardp valid_char32hexlower?(c) when c in ?a..?v or c in ?0..?9
defguardp valid_char32hexmixed?(c) when c in ?A..?V or c in ?a..?v or c in ?0..?9
# base64 alphabets have 3 ranges (A-Z, a-z, 0-9) + 2 singletons. Singletons
# are excluded from SWAR; chunks containing them fall to per-byte. Order:
# letters most common (~82%), digits (~16%), singletons (~3%).
defguardp valid_char64base?(c)
when c in ?A..?Z or c in ?a..?z or c in ?0..?9 or c == ?+ or c == ?/
defguardp valid_char64url?(c)
when c in ?A..?Z or c in ?a..?z or c in ?0..?9 or c == ?- or c == ?_
# SWAR 7-byte word validity. One guard per (encoding, case).
#
# Structure:
# 1. ASCII gate `band(w, MASK80) == 0`:
# every byte < 0x80 so the additions below cannot carry across lanes.
#
# 2. "Each byte is in range A OR range B (OR range C)" gate:
# For each valid range [lo, hi], one SWAR mask
# (w + @swar_ge_lo) bxor (w + @swar_gt_hi)
# has high bit = 1 in lanes where the byte is in [lo, hi]. OR the
# per-range masks, AND with MASK80, demand all 7 high bits are set.
defguardp valid_word16upper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_F)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word16lower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_a, w + @swar_gt_f)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word16mixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_F)
),
bxor(w + @swar_ge_a, w + @swar_gt_f)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32upper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32lower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_a, w + @swar_gt_z),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32mixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_2, w + @swar_gt_7)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexupper?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_V)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexlower?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_a, w + @swar_gt_v)
),
@swar_mask80
) == @swar_mask80
defguardp valid_word32hexmixed?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bxor(w + @swar_ge_0, w + @swar_gt_9),
bxor(w + @swar_ge_A, w + @swar_gt_V)
),
bxor(w + @swar_ge_a, w + @swar_gt_v)
),
@swar_mask80
) == @swar_mask80
# SWAR for base64 standard: 3 ranges OR'd with a single Mycroft singleton
# for '+'. The digit range is extended to [0x2F, 0x39] so it absorbs '/'
# (0x2F) — Lemire-style range/singleton merge.
defguardp valid_word64base?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_slash, w + @swar_gt_9)
),
bxor(w, @swar_plus_x7) - @swar_mask01
),
@swar_mask80
) == @swar_mask80
defguardp valid_word64url?(w)
when band(w, @swar_mask80) == 0 and
band(
bor(
bor(
bor(
bxor(w + @swar_ge_A, w + @swar_gt_Z),
bxor(w + @swar_ge_a, w + @swar_gt_z)
),
bxor(w + @swar_ge_0, w + @swar_gt_9)
),
bor(
bxor(w, @swar_dash_x7) - @swar_mask01,
bxor(w, @swar_under_x7) - @swar_mask01
)
),
@swar_mask80
) == @swar_mask80
# =========================================================================
# base16
# =========================================================================
def valid16?(string, opts \\ [])
def valid16?(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do
case Keyword.get(opts, :case, :upper) do
:upper -> validate16upper?(string)
:lower -> validate16lower?(string)
:mixed -> validate16mixed?(string)
end
end
def valid16?(string, _opts) when is_binary(string), do: false
# --- :upper -------------------------------------------------------------
defp validate16upper?(<<w::56, rest::binary>>),
do: valid_word16upper?(w) and validate16upper?(rest)
defp validate16upper?(<<>>), do: true
defp validate16upper?(<<char, rest::binary>>),
do: valid_char16upper?(char) and validate16upper?(rest)
# --- :lower -------------------------------------------------------------
defp validate16lower?(<<w::56, rest::binary>>),
do: valid_word16lower?(w) and validate16lower?(rest)
defp validate16lower?(<<>>), do: true
defp validate16lower?(<<char, rest::binary>>),
do: valid_char16lower?(char) and validate16lower?(rest)
# --- :mixed -------------------------------------------------------------
defp validate16mixed?(<<w::56, rest::binary>>),
do: valid_word16mixed?(w) and validate16mixed?(rest)
defp validate16mixed?(<<>>), do: true
defp validate16mixed?(<<char, rest::binary>>),
do: valid_char16mixed?(char) and validate16mixed?(rest)
# =========================================================================
# base32
# =========================================================================
def valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32upper?(string, pad?)
:lower -> validate32lower?(string, pad?)
:mixed -> validate32mixed?(string, pad?)
end
end
def hex_valid32?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
case Keyword.get(opts, :case, :upper) do
:upper -> validate32hexupper?(string, pad?)
:lower -> validate32hexlower?(string, pad?)
:mixed -> validate32hexmixed?(string, pad?)
end
end
# The base32 stride mirrors Baseline: split into `main` (multiple of 8 bytes)
# and a `rest` (1-8 bytes) so the last block's padding patterns are handled
# per-byte. SWAR only fast-paths the `main` loop. Same machinery covers
# both the regular alphabet (`:upper`/`:lower`/`:mixed`) and the
# extended-hex alphabet (`:hexupper`/`:hexlower`/`:hexmixed`).
for {case_name, char_guard, word_guard} <- [
{:upper, :valid_char32upper?, :valid_word32upper?},
{:lower, :valid_char32lower?, :valid_word32lower?},
{:mixed, :valid_char32mixed?, :valid_word32mixed?},
{:hexupper, :valid_char32hexupper?, :valid_word32hexupper?},
{:hexlower, :valid_char32hexlower?, :valid_word32hexlower?},
{:hexmixed, :valid_char32hexmixed?, :valid_word32hexmixed?}
] do
validate_name = :"validate32#{case_name}?"
validate_main_name = :"validate_main32#{case_name}?"
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=, ?=, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, c4, ?=, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5, ?=, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7) and unquote(char_guard)(c8)
<<c1, c2>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, c4>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
_ ->
false
end
end
defp unquote(validate_main_name)(<<w::56, rest::binary>>),
do: unquote(word_guard)(w) and unquote(validate_main_name)(rest)
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<char, rest::binary>>),
do: unquote(char_guard)(char) and unquote(validate_main_name)(rest)
end
# =========================================================================
# base64
# =========================================================================
defp remove_ignored(string, nil), do: string
defp remove_ignored(string, :whitespace) do
for <<char::8 <- string>>, char not in ~c"\s\t\r\n", into: <<>>, do: <<char::8>>
end
def valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64base?(pad?)
end
def url_valid64?(string, opts \\ []) when is_binary(string) do
pad? = Keyword.get(opts, :padding, true)
string |> remove_ignored(opts[:ignore]) |> validate64url?(pad?)
end
# Same dispatch shape as base32: split into `main` (multiple of 8 bytes) and
# `rest` (≤8 bytes containing padding). SWAR includes singletons via
# Mycroft, so no per-byte fallback is needed in the main loop.
for {variant, char_guard, word_guard} <- [
{:base, :valid_char64base?, :valid_word64base?},
{:url, :valid_char64url?, :valid_word64url?}
] do
validate_name = :"validate64#{variant}?"
validate_main_name = :"validate_main64#{variant}?"
defp unquote(validate_name)(<<>>, _pad?), do: true
defp unquote(validate_name)(string, pad?) do
segs = div(byte_size(string) + 7, 8) - 1
<<main::size(^segs)-binary-unit(64), rest::binary>> = string
main_valid? = unquote(validate_main_name)(main)
case rest do
_ when not main_valid? ->
false
<<c1, c2, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and unquote(char_guard)(c3)
<<c1, c2, c3, c4>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4)
<<c1, c2, c3, c4, c5, c6, ?=, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6)
<<c1, c2, c3, c4, c5, c6, c7, ?=>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
<<c1, c2, c3, c4, c5, c6, c7, c8>> ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7) and unquote(char_guard)(c8)
<<c1, c2>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2)
<<c1, c2, c3>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and unquote(char_guard)(c3)
<<c1, c2, c3, c4, c5, c6>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6)
<<c1, c2, c3, c4, c5, c6, c7>> when not pad? ->
unquote(char_guard)(c1) and unquote(char_guard)(c2) and
unquote(char_guard)(c3) and unquote(char_guard)(c4) and
unquote(char_guard)(c5) and unquote(char_guard)(c6) and
unquote(char_guard)(c7)
_ ->
false
end
end
defp unquote(validate_main_name)(<<w::56, rest::binary>>),
do: unquote(word_guard)(w) and unquote(validate_main_name)(rest)
defp unquote(validate_main_name)(<<>>), do: true
defp unquote(validate_main_name)(<<char, rest::binary>>),
do: unquote(char_guard)(char) and unquote(validate_main_name)(rest)
end
end
sizes = [
{"1KiB", 1 * 1024},
{"100KiB", 100 * 1024},
{"1MiB", 1024 * 1024}
]
# Each input is `{binary, case_opt}` so both scenarios (Baseline / Optimised)
# call the same function with the same arguments and Benchee shows them
# head-to-head per input row.
inputs =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.encode16(data),
lower = Base.encode16(data, case: :lower),
# Mixed-case: flip every other letter to lowercase.
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?F and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path shapes (upper only; 'Z' is invalid in all three modes anyway).
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "Z",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"Z" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
# Sanity-check Baseline and Optimised against the stdlib Base for every input.
for {label, {bin, case_opt}} <- inputs do
ref = Base.valid16?(bin, case: case_opt)
b = Baseline.valid16?(bin, case: case_opt)
o = Optimised.valid16?(bin, case: case_opt)
ref === b ||
raise "Baseline mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid16? equivalence check passed.\n")
# --- base32 inputs ------------------------------------------------------
inputs32 =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.encode32(data),
lower = Base.encode32(data, case: :lower),
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?Z and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path: '!' (0x21) is invalid in every base32 alphabet.
# Replacing the last char may overwrite a `=` pad byte; that's fine —
# both impls then fall through their `case rest do` to `_ -> false`.
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "!",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"!" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
for {label, {bin, case_opt}} <- inputs32 do
ref = Base.valid32?(bin, case: case_opt)
b = Baseline.valid32?(bin, case: case_opt)
o = Optimised.valid32?(bin, case: case_opt)
ref === b ||
raise "Baseline valid32? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised valid32? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid32? equivalence check passed.\n")
# --- base32 hex inputs --------------------------------------------------
inputs32hex =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
upper = Base.hex_encode32(data),
lower = Base.hex_encode32(data, case: :lower),
mixed =
upper
|> :binary.bin_to_list()
|> Enum.with_index()
|> Enum.map(fn
{c, i} when c in ?A..?V and rem(i, 2) == 1 -> c - ?A + ?a
{c, _} -> c
end)
|> :binary.list_to_bin(),
# Slow-path: '!' (0x21) is invalid in every base32hex alphabet too.
invalid_at_end = binary_part(upper, 0, byte_size(upper) - 1) <> "!",
mid = div(byte_size(upper), 2),
invalid_at_mid =
binary_part(upper, 0, mid) <>
"!" <> binary_part(upper, mid + 1, byte_size(upper) - mid - 1),
{label, payload} <- [
{"#{size_label} upper valid", {upper, :upper}},
{"#{size_label} lower valid", {lower, :lower}},
{"#{size_label} mixed valid", {mixed, :mixed}},
{"#{size_label} upper invalid@end", {invalid_at_end, :upper}},
{"#{size_label} upper invalid@mid", {invalid_at_mid, :upper}}
],
into: %{},
do: {label, payload}
for {label, {bin, case_opt}} <- inputs32hex do
ref = Base.hex_valid32?(bin, case: case_opt)
b = Baseline.hex_valid32?(bin, case: case_opt)
o = Optimised.hex_valid32?(bin, case: case_opt)
ref === b ||
raise "Baseline hex_valid32? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised hex_valid32? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# hex_valid32? equivalence check passed.\n")
# --- base64 inputs ------------------------------------------------------
#
# Encoding random bytes via Base.encode64/url_encode64 yields a uniform
# distribution over the 64-char alphabet — each character position has a
# 2/64 = 3.125% chance of being a "singleton" (`+`/`/` for base, `-`/`_`
# for url). This matches the real-world distribution of base64-encoded
# binary data (random/compressed payloads, tokens, etc.).
#
# We also generate an "alnum valid" variant: a synthetic input drawn only
# from `A-Z, a-z, 0-9` (0% singletons). This isolates how much of the
# overall cost depends on singletons specifically — useful when comparing
# "include singletons in SWAR" vs "alnum-only SWAR + scalar fallback".
alnum64_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
alnum_pad_to_4 = fn binary ->
case rem(byte_size(binary), 4) do
0 -> binary
1 -> binary_part(binary, 0, byte_size(binary) - 1)
2 -> binary <> "=="
3 -> binary <> "="
end
end
alnum_string = fn target_len ->
for(_ <- 1..target_len, into: <<>>, do: <<Enum.random(alnum64_alphabet)>>)
|> alnum_pad_to_4.()
end
inputs64base =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
base = Base.encode64(data),
alnum = alnum_string.(byte_size(base)),
# Slow-path: '!' (0x21) is invalid in every base64 alphabet.
invalid_at_end = binary_part(base, 0, byte_size(base) - 1) <> "!",
mid = div(byte_size(base), 2),
invalid_at_mid =
binary_part(base, 0, mid) <>
"!" <> binary_part(base, mid + 1, byte_size(base) - mid - 1),
{label, payload} <- [
{"#{size_label} valid (random, ~3.1% +/)", base},
{"#{size_label} alnum valid (0% +/)", alnum},
{"#{size_label} invalid@end", invalid_at_end},
{"#{size_label} invalid@mid", invalid_at_mid}
],
into: %{},
do: {label, payload}
inputs64url =
for {size_label, n} <- sizes,
data = :crypto.strong_rand_bytes(n),
url = Base.url_encode64(data),
alnum = alnum_string.(byte_size(url)),
invalid_at_end = binary_part(url, 0, byte_size(url) - 1) <> "!",
mid = div(byte_size(url), 2),
invalid_at_mid =
binary_part(url, 0, mid) <>
"!" <> binary_part(url, mid + 1, byte_size(url) - mid - 1),
{label, payload} <- [
{"#{size_label} valid (random, ~3.1% -_)", url},
{"#{size_label} alnum valid (0% -_)", alnum},
{"#{size_label} invalid@end", invalid_at_end},
{"#{size_label} invalid@mid", invalid_at_mid}
],
into: %{},
do: {label, payload}
# Sanity-print singleton density on the 1 MiB samples so the bench output
# carries a self-documenting reminder of what "valid" actually contains.
_ =
for {label, bin} <- inputs64base,
String.starts_with?(label, "1MiB valid"),
do:
(
count =
Enum.count(:binary.bin_to_list(bin), fn c -> c == ?+ or c == ?/ end)
pct = Float.round(count / byte_size(bin) * 100, 3)
IO.puts("# inputs64base #{label}: #{count} +/ in #{byte_size(bin)} bytes (#{pct}%)")
)
for {label, bin} <- inputs64base do
ref = Base.valid64?(bin)
b = Baseline.valid64?(bin)
o = Optimised.valid64?(bin)
ref === b ||
raise "Baseline valid64? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised valid64? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
for {label, bin} <- inputs64url do
ref = Base.url_valid64?(bin)
b = Baseline.url_valid64?(bin)
o = Optimised.url_valid64?(bin)
ref === b ||
raise "Baseline url_valid64? mismatch on #{label}: stdlib=#{ref} baseline=#{b}"
ref === o ||
raise "Optimised url_valid64? mismatch on #{label}: stdlib=#{ref} optimised=#{o}"
end
IO.puts("# valid64? / url_valid64? equivalence check passed.\n")
if run?.("valid16") do
IO.puts("\n========== valid16? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.valid16?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.valid16?(s, case: c) end
},
inputs: inputs,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("valid32") do
IO.puts("\n========== valid32? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.valid32?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.valid32?(s, case: c) end
},
inputs: inputs32,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("hex_valid32") do
IO.puts("\n========== hex_valid32? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn {s, c} -> Baseline.hex_valid32?(s, case: c) end,
"Optimised" => fn {s, c} -> Optimised.hex_valid32?(s, case: c) end
},
inputs: inputs32hex,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("valid64") do
IO.puts("\n========== valid64? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn s -> Baseline.valid64?(s) end,
"Optimised" => fn s -> Optimised.valid64?(s) end
},
inputs: inputs64base,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end
if run?.("url_valid64") do
IO.puts("\n========== url_valid64? benchmark ==========\n")
Benchee.run(
%{
"Baseline" => fn s -> Baseline.url_valid64?(s) end,
"Optimised" => fn s -> Optimised.url_valid64?(s) end
},
inputs: inputs64url,
warmup: 2,
time: 5,
print: [fast_warning: false, benchmarking: false]
)
end |
PJUllrich
changed the title
Base.valid16? and Base.valid32?Base validations
Member
|
💚 💙 💜 💛 ❤️ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
I applied a SWAR (SIMD Within A Register) approach to all
Base.validX?functions and benchmarked them against the baseline, but could find performance improvements of roughly 25% only forvalid16?andvalid32?. My assumption is that in the64case, there are simply too many operations in a single SWAR procedure which makes it slower than the current tuple lookup. Weirdly, I could not find any improvements inhex_valid32?although it has the same number of range checks asvalid32?.I will post the benchmark and results in separate comments below.
EDIT:
The following PRs inspired this work:
SWAR-optimize ASCII fast-path: #15255
Use 56-bit SWAR to accelerate binary ASCII traversal: erlang/otp#10948