Skip to main content

nautilus_core/string/
urlencoding.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! URL percent-encoding and decoding per [RFC 3986].
17//!
18//! The unreserved set is `ALPHA / DIGIT / "-" / "." / "_" / "~"`; every other
19//! byte is percent-encoded as `%HH` using uppercase hexadecimal as recommended
20//! by [RFC 3986 Section 2.1].
21//!
22//! Decoding accepts both uppercase and lowercase hex. A `%` that is not
23//! followed by two hex digits is passed through literally, matching the
24//! behavior of the `urlencoding` crate that this module replaces.
25//!
26//! [RFC 3986]: https://datatracker.ietf.org/doc/html/rfc3986
27//! [RFC 3986 Section 2.1]: https://datatracker.ietf.org/doc/html/rfc3986#section-2.1
28
29use std::{borrow::Cow, string::FromUtf8Error};
30
31use thiserror::Error;
32
33const UNRESERVED: [bool; 256] = {
34    let mut table = [false; 256];
35    let mut i = b'0';
36    while i <= b'9' {
37        table[i as usize] = true;
38        i += 1;
39    }
40    i = b'A';
41    while i <= b'Z' {
42        table[i as usize] = true;
43        i += 1;
44    }
45    i = b'a';
46    while i <= b'z' {
47        table[i as usize] = true;
48        i += 1;
49    }
50    table[b'-' as usize] = true;
51    table[b'.' as usize] = true;
52    table[b'_' as usize] = true;
53    table[b'~' as usize] = true;
54    table
55};
56
57const ENCODE_PAIR: [[u8; 2]; 256] = {
58    const NIBBLE: [u8; 16] = *b"0123456789ABCDEF";
59    let mut table = [[0u8; 2]; 256];
60    let mut i = 0u16;
61    while i < 256 {
62        table[i as usize] = [NIBBLE[(i >> 4) as usize], NIBBLE[(i & 0x0f) as usize]];
63        i += 1;
64    }
65    table
66};
67
68// 0xFF sentinel marks non-hex characters
69const DECODE_NIBBLE: [u8; 256] = {
70    let mut table = [0xFFu8; 256];
71    let mut i = 0u8;
72    while i < 10 {
73        table[(b'0' + i) as usize] = i;
74        i += 1;
75    }
76    i = 0;
77    while i < 6 {
78        table[(b'a' + i) as usize] = 10 + i;
79        table[(b'A' + i) as usize] = 10 + i;
80        i += 1;
81    }
82    table
83};
84
85/// Percent-encodes a string per RFC 3986.
86///
87/// Returns the input borrowed when every byte is already in the unreserved
88/// set, otherwise an owned encoded copy.
89///
90/// [`encode_bytes`] only emits ASCII bytes (unreserved characters or `%HH` pairs), so
91/// [`String::from_utf8`] always succeeds.
92#[allow(
93    clippy::missing_panics_doc,
94    reason = "percent-encoded output contains only ASCII bytes"
95)]
96#[must_use]
97pub fn encode(input: &str) -> Cow<'_, str> {
98    match encode_bytes(input.as_bytes()) {
99        Cow::Borrowed(_) => Cow::Borrowed(input),
100        Cow::Owned(bytes) => Cow::Owned(String::from_utf8(bytes).expect("encoded output is ASCII")),
101    }
102}
103
104/// Percent-encodes a byte slice per RFC 3986.
105///
106/// Returns the input borrowed when every byte is already in the unreserved
107/// set, otherwise an owned encoded copy.
108#[must_use]
109pub fn encode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
110    let Some(first) = input.iter().position(|&b| !UNRESERVED[b as usize]) else {
111        return Cow::Borrowed(input);
112    };
113
114    // Slack for payloads dominated by reserved chars without over-allocating
115    // on mostly-unreserved inputs; Vec's geometric growth covers the rest.
116    let mut out = Vec::with_capacity(input.len() + input.len() / 2 + 16);
117    out.extend_from_slice(&input[..first]);
118
119    let mut rest = &input[first..];
120    while let Some(&byte) = rest.first() {
121        if UNRESERVED[byte as usize] {
122            let run_end = rest
123                .iter()
124                .position(|&b| !UNRESERVED[b as usize])
125                .unwrap_or(rest.len());
126            out.extend_from_slice(&rest[..run_end]);
127            rest = &rest[run_end..];
128        } else {
129            out.push(b'%');
130            out.extend_from_slice(&ENCODE_PAIR[byte as usize]);
131            rest = &rest[1..];
132        }
133    }
134    Cow::Owned(out)
135}
136
137/// Percent-decodes a string per RFC 3986.
138///
139/// Returns the input borrowed when no `%` is present. Otherwise decodes
140/// `%HH` pairs (hex is case-insensitive) and leaves any `%` that is not
141/// followed by two hex digits in place.
142///
143/// # Errors
144///
145/// Returns an error if the decoded bytes are not valid UTF-8.
146pub fn decode(input: &str) -> Result<Cow<'_, str>, DecodeError> {
147    match decode_bytes(input.as_bytes()) {
148        Cow::Borrowed(_) => Ok(Cow::Borrowed(input)),
149        Cow::Owned(bytes) => String::from_utf8(bytes)
150            .map(Cow::Owned)
151            .map_err(DecodeError::InvalidUtf8),
152    }
153}
154
155/// Percent-decodes a byte slice.
156///
157/// Returns the input borrowed when no `%` is present. A `%` that is not
158/// followed by two hex digits is left in place.
159#[must_use]
160pub fn decode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
161    let Some(first) = input.iter().position(|&b| b == b'%') else {
162        return Cow::Borrowed(input);
163    };
164
165    let mut out = Vec::with_capacity(input.len());
166    out.extend_from_slice(&input[..first]);
167
168    let mut i = first;
169    while i < input.len() {
170        if input[i] != b'%' {
171            let run_start = i;
172            while i < input.len() && input[i] != b'%' {
173                i += 1;
174            }
175            out.extend_from_slice(&input[run_start..i]);
176            continue;
177        }
178
179        if i + 2 < input.len() {
180            let hi = DECODE_NIBBLE[input[i + 1] as usize];
181            let lo = DECODE_NIBBLE[input[i + 2] as usize];
182            if (hi | lo) & 0xF0 == 0 {
183                out.push((hi << 4) | lo);
184                i += 3;
185                continue;
186            }
187        }
188
189        // Malformed or trailing `%`: pass through literally
190        out.push(b'%');
191        i += 1;
192    }
193    Cow::Owned(out)
194}
195
196/// Errors from URL percent-decoding.
197#[derive(Debug, Error)]
198pub enum DecodeError {
199    /// Decoded bytes are not valid UTF-8.
200    #[error("invalid UTF-8 in decoded bytes: {0}")]
201    InvalidUtf8(#[from] FromUtf8Error),
202}
203
204#[cfg(test)]
205mod tests {
206    use proptest::prelude::*;
207    use rstest::rstest;
208
209    use super::*;
210
211    // RFC 3986 Section 2.3: unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
212    const UNRESERVED_CHARS: &str =
213        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~";
214
215    // RFC 3986 Section 2.2: reserved chars (gen-delims + sub-delims) must be
216    // percent-encoded when used as data.
217    const RESERVED_CHARS: &str = ":/?#[]@!$&'()*+,;=";
218
219    #[rstest]
220    #[case("", "")]
221    #[case("abc", "abc")]
222    #[case("ABC-xyz_0.9~", "ABC-xyz_0.9~")]
223    #[case(" ", "%20")]
224    #[case("+", "%2B")]
225    #[case("/", "%2F")]
226    #[case("?", "%3F")]
227    #[case("#", "%23")]
228    #[case("&", "%26")]
229    #[case("=", "%3D")]
230    #[case("%", "%25")]
231    #[case("hello world", "hello%20world")]
232    #[case("a b+c/d", "a%20b%2Bc%2Fd")]
233    // Uppercase hex per RFC 3986 Section 2.1
234    #[case("\x7f", "%7F")]
235    fn test_encode_ascii_vectors(#[case] input: &str, #[case] expected: &str) {
236        assert_eq!(encode(input), expected);
237    }
238
239    #[rstest]
240    fn test_encode_all_unreserved_unchanged() {
241        // Every char in the unreserved set should pass through.
242        let out = encode(UNRESERVED_CHARS);
243        assert_eq!(out, UNRESERVED_CHARS);
244        // And the Cow should be Borrowed (zero-copy).
245        assert!(matches!(out, Cow::Borrowed(_)));
246    }
247
248    #[rstest]
249    fn test_encode_all_reserved_percent_encoded() {
250        let out = encode(RESERVED_CHARS);
251        // Each of the 18 chars becomes a 3-byte `%HH` sequence.
252        assert_eq!(out.len(), RESERVED_CHARS.len() * 3);
253        // None of the unreserved chars, `%`, or digits A-F should appear raw
254        // in the output except as part of a `%HH` triple.
255        for byte in out.bytes() {
256            assert!(
257                matches!(byte, b'%' | b'0'..=b'9' | b'A'..=b'F'),
258                "unexpected byte {byte:#04x} in encoded reserved output"
259            );
260        }
261    }
262
263    #[rstest]
264    fn test_encode_hex_is_uppercase() {
265        // Verify RFC 3986 Section 2.1: producers SHOULD emit uppercase hex.
266        let out = encode("/");
267        assert_eq!(out, "%2F");
268        assert!(!out.contains('f'));
269    }
270
271    #[rstest]
272    fn test_encode_every_byte_position() {
273        // For each byte 0x00..=0xFF, encode a one-byte slice and verify that
274        // the output matches the spec expectation.
275        for byte in 0u8..=255 {
276            let input = [byte];
277            let out = encode_bytes(&input);
278
279            if UNRESERVED[byte as usize] {
280                assert!(
281                    matches!(out, Cow::Borrowed(_)),
282                    "unreserved byte {byte:#04x} should not allocate"
283                );
284                assert_eq!(out.as_ref(), &[byte]);
285            } else {
286                let expected = format!("%{byte:02X}").into_bytes();
287                assert_eq!(out.as_ref(), expected.as_slice(), "byte {byte:#04x}");
288            }
289        }
290    }
291
292    #[rstest]
293    fn test_encode_utf8_multibyte() {
294        // U+00E9 encoded as UTF-8 is `0xC3 0xA9` (two bytes).
295        assert_eq!(encode("\u{00E9}"), "%C3%A9");
296        // U+4E2D encoded as UTF-8 is `0xE4 0xB8 0xAD` (three bytes).
297        assert_eq!(encode("\u{4E2D}"), "%E4%B8%AD");
298        // Grinning face emoji U+1F600 is `0xF0 0x9F 0x98 0x80` (four bytes).
299        assert_eq!(encode("\u{1F600}"), "%F0%9F%98%80");
300    }
301
302    #[rstest]
303    fn test_encode_mixed_ascii_and_utf8() {
304        assert_eq!(encode("a é/"), "a%20%C3%A9%2F");
305    }
306
307    #[rstest]
308    fn test_encode_returns_borrowed_when_no_work() {
309        let out = encode("safe-string_123.xyz~");
310        assert!(matches!(out, Cow::Borrowed(_)));
311    }
312
313    #[rstest]
314    fn test_encode_returns_owned_when_encoding_needed() {
315        let out = encode("needs encoding");
316        assert!(matches!(out, Cow::Owned(_)));
317    }
318
319    #[rstest]
320    #[case("", "")]
321    #[case("abc", "abc")]
322    #[case("%20", " ")]
323    #[case("%2F", "/")]
324    #[case("%2f", "/")] // lowercase hex must be accepted
325    #[case("%2b", "+")]
326    #[case("%25", "%")]
327    #[case("hello%20world", "hello world")]
328    #[case("a%20b%2Bc%2Fd", "a b+c/d")]
329    #[case("%C3%A9", "\u{00E9}")]
330    #[case("%E4%B8%AD", "\u{4E2D}")]
331    #[case("%F0%9F%98%80", "\u{1F600}")]
332    fn test_decode_ascii_and_utf8_vectors(#[case] input: &str, #[case] expected: &str) {
333        assert_eq!(decode(input).unwrap(), expected);
334    }
335
336    #[rstest]
337    #[case("%", "%")] // bare `%` at end passes through
338    #[case("%2", "%2")] // one hex digit at end
339    #[case("%GG", "%GG")] // non-hex digits
340    #[case("%2G", "%2G")] // second nibble invalid
341    #[case("%G2", "%G2")] // first nibble invalid
342    #[case("%%20", "% ")] // first `%` literal, then `%20` decodes
343    #[case("100%", "100%")] // `%` at end after ASCII
344    fn test_decode_malformed_percent_passes_through(#[case] input: &str, #[case] expected: &str) {
345        assert_eq!(decode(input).unwrap(), expected);
346    }
347
348    #[rstest]
349    fn test_decode_returns_borrowed_when_no_percent() {
350        let out = decode("no-percent-here").unwrap();
351        assert!(matches!(out, Cow::Borrowed(_)));
352    }
353
354    #[rstest]
355    fn test_decode_returns_owned_when_percent_present() {
356        let out = decode("a%20b").unwrap();
357        assert!(matches!(out, Cow::Owned(_)));
358    }
359
360    #[rstest]
361    #[case("this%2x%26that", "this%2x&that")]
362    #[case("%%25", "%%")]
363    #[case("%2%26", "%2&")]
364    #[case("a%2Zb%20c", "a%2Zb c")]
365    #[case("%:0", "%:0")]
366    fn test_decode_malformed_then_valid(#[case] input: &str, #[case] expected: &str) {
367        assert_eq!(decode(input).unwrap(), expected);
368    }
369
370    #[rstest]
371    fn test_decode_invalid_utf8_errors() {
372        // `0xFF` is not valid UTF-8 on its own.
373        let err = decode("%FF").unwrap_err();
374        assert!(matches!(err, DecodeError::InvalidUtf8(_)));
375    }
376
377    #[rstest]
378    fn test_decode_invalid_utf8_bytes_ok() {
379        // `decode_bytes` does not validate UTF-8.
380        let out = decode_bytes(b"%FF");
381        assert_eq!(out.as_ref(), &[0xFF]);
382    }
383
384    #[rstest]
385    fn test_decode_consecutive_percent_triples() {
386        // Three consecutive `%HH` sequences decoding multi-byte UTF-8.
387        assert_eq!(decode("%e2%98%83").unwrap(), "\u{2603}"); // snowman U+2603
388    }
389
390    #[rstest]
391    fn test_decode_nul_byte() {
392        // `%00` decodes to the NUL byte, which is valid UTF-8 (U+0000).
393        let decoded = decode("a%00b").unwrap();
394        assert_eq!(decoded.as_bytes(), &[b'a', 0x00, b'b']);
395    }
396
397    #[rstest]
398    fn test_roundtrip_every_byte() {
399        // For every byte 0x00..=0xFF, encoding then decoding must recover
400        // the original byte exactly.
401        for byte in 0u8..=255 {
402            let input = [byte];
403            let encoded = encode_bytes(&input);
404            let decoded = decode_bytes(encoded.as_ref());
405            assert_eq!(
406                decoded.as_ref(),
407                input.as_slice(),
408                "round-trip failed for byte {byte:#04x}"
409            );
410        }
411    }
412
413    #[rstest]
414    #[case("hello")]
415    #[case("a b c")]
416    #[case("https://example.com/path?q=1&x=2")]
417    #[case("\u{00E9}\u{00E0}\u{00FC}")]
418    #[case("\u{4E2D}\u{6587}\u{6D4B}\u{8BD5}")]
419    #[case("mix 123 !@# %^&*()")]
420    #[case("\u{1F600}\u{1F680}\u{1F3C6}")]
421    fn test_roundtrip_string(#[case] input: &str) {
422        let encoded = encode(input);
423        let decoded = decode(&encoded).unwrap();
424        assert_eq!(decoded, input);
425    }
426
427    #[rstest]
428    fn test_encoded_output_only_ascii() {
429        // Encoded output must always be pure ASCII (unreserved bytes + `%HH`).
430        let encoded = encode("\u{00E9}\u{4E2D}\u{1F600}");
431        assert!(encoded.is_ascii(), "encoded output must be ASCII-only");
432    }
433
434    #[rstest]
435    fn test_encode_bytes_arbitrary_binary() {
436        // Encoding arbitrary bytes (including non-UTF-8) yields a valid
437        // percent-encoded ASCII sequence.
438        let input: Vec<u8> = (0u8..=255).collect();
439        let encoded = encode_bytes(&input);
440        assert!(encoded.iter().all(u8::is_ascii));
441        let decoded = decode_bytes(encoded.as_ref());
442        assert_eq!(decoded.as_ref(), input.as_slice());
443    }
444
445    #[rstest]
446    fn test_decode_error_display_and_source() {
447        let err = decode("%FF").unwrap_err();
448        let msg = err.to_string();
449        assert!(msg.starts_with("invalid UTF-8"), "got: {msg}");
450        assert!(std::error::Error::source(&err).is_some());
451    }
452
453    // Independent reference implementation used to cross-check our tuned
454    // implementation on random inputs. Pure-Rust, loop-based, no table
455    // lookups: if both agree across thousands of random inputs we have
456    // strong evidence the tuned version is spec-correct.
457    fn reference_encode(input: &[u8]) -> Vec<u8> {
458        let mut out = Vec::with_capacity(input.len());
459        for &b in input {
460            let is_unreserved =
461                b.is_ascii_alphanumeric() || b == b'-' || b == b'.' || b == b'_' || b == b'~';
462            if is_unreserved {
463                out.push(b);
464            } else {
465                out.push(b'%');
466                out.extend_from_slice(format!("{b:02X}").as_bytes());
467            }
468        }
469        out
470    }
471
472    fn reference_decode(input: &[u8]) -> Vec<u8> {
473        let mut out = Vec::with_capacity(input.len());
474        let mut i = 0;
475        while i < input.len() {
476            if input[i] == b'%' && i + 2 < input.len() {
477                let a = input[i + 1];
478                let b = input[i + 2];
479                if a.is_ascii_hexdigit() && b.is_ascii_hexdigit() {
480                    let hi = if a.is_ascii_digit() {
481                        a - b'0'
482                    } else {
483                        (a | 0x20) - b'a' + 10
484                    };
485                    let lo = if b.is_ascii_digit() {
486                        b - b'0'
487                    } else {
488                        (b | 0x20) - b'a' + 10
489                    };
490                    out.push((hi << 4) | lo);
491                    i += 3;
492                    continue;
493                }
494            }
495            out.push(input[i]);
496            i += 1;
497        }
498        out
499    }
500
501    fn malformed_percent_sequence() -> impl Strategy<Value = Vec<u8>> {
502        prop_oneof![
503            Just(vec![b'%']),
504            (any::<u8>(), any::<u8>())
505                .prop_filter("contains a non-hex byte", |(hi, lo)| {
506                    !hi.is_ascii_hexdigit() || !lo.is_ascii_hexdigit()
507                })
508                .prop_map(|(hi, lo)| vec![b'%', hi, lo]),
509        ]
510    }
511
512    proptest::proptest! {
513        #[rstest]
514        fn prop_encode_matches_reference(input: Vec<u8>) {
515            let actual = encode_bytes(&input);
516            let expected = reference_encode(&input);
517            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
518        }
519
520        #[rstest]
521        fn prop_decode_matches_reference(input: Vec<u8>) {
522            let actual = decode_bytes(&input);
523            let expected = reference_decode(&input);
524            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
525        }
526
527        #[rstest]
528        fn prop_malformed_percent_sequences_match_reference(
529            prefix in proptest::collection::vec(any::<u8>(), 0..16),
530            malformed in malformed_percent_sequence(),
531            suffix in proptest::collection::vec(any::<u8>(), 0..16),
532        ) {
533            let mut input = prefix;
534            input.extend(malformed);
535            input.extend(suffix);
536
537            let actual = decode_bytes(&input);
538            let expected = reference_decode(&input);
539            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
540        }
541
542        #[rstest]
543        fn prop_bytes_roundtrip(input: Vec<u8>) {
544            let encoded = encode_bytes(&input);
545            let decoded = decode_bytes(encoded.as_ref());
546            proptest::prop_assert_eq!(decoded.as_ref(), input.as_slice());
547        }
548
549        #[rstest]
550        fn prop_string_roundtrip(input: String) {
551            let encoded = encode(&input);
552            let decoded = decode(&encoded).unwrap();
553            proptest::prop_assert_eq!(decoded.as_ref(), input.as_str());
554        }
555
556        #[rstest]
557        fn prop_encoded_output_ascii(input: Vec<u8>) {
558            let encoded = encode_bytes(&input);
559            proptest::prop_assert!(encoded.iter().all(u8::is_ascii));
560        }
561    }
562}