File 3421-stdlib-add-quote-unquote-functions-in-uri_... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 3421-stdlib-add-quote-unquote-functions-in-uri_string.patch of Package erlang (Revision 06895323ebe8e4f43b9b3d18d00c12c8)

Currently displaying revision 06895323ebe8e4f43b9b3d18d00c12c8 , Show latest

From 3d980d91c8fc81b02cd5cf85595bbf569ff9652e Mon Sep 17 00:00:00 2001
From: Jakub Witczak <kuba@erlang.org>
Date: Wed, 9 Feb 2022 16:34:37 +0100
Subject: [PATCH 1/2] stdlib: add quote, unquote functions in uri_string

- functions for percent encoding and decoding user data
- intended as utility functions for preparing URI components
---
 lib/common_test/test/erl2html2_SUITE.erl |   4 +-
 lib/inets/src/http_lib/http_uri.erl      |   4 +-
 lib/inets/src/http_server/httpd_util.erl |  11 +-
 lib/inets/src/inets_app/inets.app.src    |   2 +-
 lib/stdlib/doc/src/uri_string.xml        |  79 ++++++++++++-
 lib/stdlib/doc/src/uri_string_usage.xml  |  11 +-
 lib/stdlib/src/uri_string.erl            |  30 ++++-
 lib/stdlib/test/uri_string_SUITE.erl     | 139 ++++++++++++++++++++++-
 8 files changed, 258 insertions(+), 22 deletions(-)

diff --git a/lib/common_test/test/erl2html2_SUITE.erl b/lib/common_test/test/erl2html2_SUITE.erl
index b2336ff0bc..bac028350a 100644
--- a/lib/common_test/test/erl2html2_SUITE.erl
+++ b/lib/common_test/test/erl2html2_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2012-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2012-2022. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -262,7 +262,7 @@ sax_event({startElement,_Uri,"a",_QN,Attrs},{Exports,LastExprFuncs,PrevLine}) ->
 	    {match,[FStr,EndStr]} =
 		 re:run(Name,"^(.*)-(last_expr|[0-9]+)$",
 			[{capture,all_but_first,list}]),
-	    F = list_to_atom(http_uri:decode(FStr)),
+	    F = list_to_atom(uri_string:unquote(FStr)),
 	    case EndStr of
 		"last_expr" ->
 		    true = lists:member(F,LastExprFuncs),
diff --git a/lib/inets/src/http_lib/http_uri.erl b/lib/inets/src/http_lib/http_uri.erl
index 5799163271..5795280635 100644
--- a/lib/inets/src/http_lib/http_uri.erl
+++ b/lib/inets/src/http_lib/http_uri.erl
@@ -64,8 +64,8 @@
 
 -deprecated({parse, 1, "use uri_string functions instead"}).
 -deprecated({parse, 2, "use uri_string functions instead"}).
--deprecated({encode, 1, "use uri_string functions instead"}).
--deprecated({decode, 1, "use uri_string functions instead"}).
+-deprecated({encode, 1, "use uri_string:quote function instead"}).
+-deprecated({decode, 1, "use uri_string:unquote function instead"}).
 -deprecated({scheme_defaults, 0, "use uri_string functions instead"}).
  
 
diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl
index 70d5e776bb..c1f273af79 100644
--- a/lib/inets/src/http_server/httpd_util.erl
+++ b/lib/inets/src/http_server/httpd_util.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2021. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2022. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -398,10 +398,11 @@ month(12) -> "Dec".
 %% decode_hex
 
 decode_hex(URI) ->
-    http_uri:decode(URI).
+    uri_string:unquote(URI).
 
 encode_hex(URI) ->
-    http_uri:encode(URI).
+    SafeChars = "!$()*", %% characters not encoded by deprecated http_uri:encode/1
+    uri_string:quote(URI, SafeChars).
 
 %% flatlength
 flatlength(List) ->
diff --git a/lib/stdlib/doc/src/uri_string.xml b/lib/stdlib/doc/src/uri_string.xml
index 4fe24e656f..08fc09e150 100644
--- a/lib/stdlib/doc/src/uri_string.xml
+++ b/lib/stdlib/doc/src/uri_string.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>2017</year><year>2020</year>
+      <year>2017</year><year>2022</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -84,9 +84,14 @@
       <item>Dissecting form-urlencoded query strings into a list of key-value pairs<br></br>
       <seemfa marker="#dissect_query/1"><c>dissect_query/1</c></seemfa>
       </item>
-      <item>Decoding percent-encoded triplets<br></br>
+      <item>Decoding percent-encoded triplets in URI map or a specific component of URI<br></br>
       <seemfa marker="#percent_decode/1"><c>percent_decode/1</c></seemfa>
       </item>
+      <item>Preparing and retrieving application specific data included in URI components<br></br>
+      <seemfa marker="#quote/1"><c>quote/1</c></seemfa>
+      <seemfa marker="#quote/2"><c>quote/2</c></seemfa>
+      <seemfa marker="#unquote/1"><c>unquote/1</c></seemfa>
+      </item>
     </list>
     <p>There are four different encodings present during the handling of URIs:</p>
     <list type="bulleted">
@@ -110,6 +115,15 @@
     character encoding and it is usually defined by the protocol or surrounding text. This library
     takes the same assumption, binary and percent-encoding are handled as one configuration unit,
     they cannot be set to different values.</p>
+    <p>Quoting functions are intended to be used by URI producing application
+    during component preparation or retrieval phase to avoid conflicts between
+    data and characters used in URI syntax. Quoting functions use percent
+    encoding, but with different rules than for example during execution of
+    <c>recompose/1</c>. It is user responsibility to provide quoting
+    functions with application data only and using their output to combine an
+    URI component.<br></br>Quoting functions can for instance be used for constructing a path
+    component with a segment containing '/' character which should not collide with
+    '/' used as general delimiter in path component.</p>
   </description>
 
   <datatypes>
@@ -357,6 +371,48 @@
       </desc>
     </func>
 
+    <func>
+      <name name="quote" arity="1" since="OTP 25.0"/>
+      <fsummary>Percent encode characters out of unreserved set.</fsummary>
+      <desc>
+        <p>Replaces characters out of unreserved set with their percent encoded equivalents.</p>
+        <p>Unreserved characters defined in
+        <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url> are not quoted.</p>
+        <p><em>Example:</em></p>
+        <pre>
+1> <input><![CDATA[uri_string:quote("SomeId/04").]]></input>
+<![CDATA["SomeId%2F04"]]>
+2> <input><![CDATA[uri_string:quote(<<"SomeId/04">>).]]></input>
+<![CDATA[<<"SomeId%2F04">>]]>
+	</pre>
+
+        <warning><p>Function is not aware about any URI component context and
+        should not be used on whole URI. If applied more than once on the same
+        data, might produce unexpected results.</p></warning>
+      </desc>
+    </func>
+
+    <func>
+      <name name="quote" arity="2" since="OTP 25.0"/>
+      <fsummary>Percent encode characters out of unreserved set and user defined
+      safe list.</fsummary>
+      <desc>
+        <p>Same as <c>quote/1</c>, but <c><anno>Safe</anno></c> allows user to
+        provide a list of characters to be protected from encoding.</p>
+        <p><em>Example:</em></p>
+        <pre>
+1> <input><![CDATA[uri_string:quote("SomeId/04", "/").]]></input>
+<![CDATA["SomeId/04"]]>
+2> <input><![CDATA[uri_string:quote(<<"SomeId/04">>, "/").]]></input>
+<![CDATA[<<"SomeId/04">>]]>
+	</pre>
+
+        <warning><p>Function is not aware about any URI component context and
+        should not be used on whole URI. If applied more than once on the same
+        data, might produce unexpected results.</p></warning>
+      </desc>
+    </func>
+
     <func>
       <name name="recompose" arity="1" since="OTP 21.0"/>
       <fsummary>Recompose URI.</fsummary>
@@ -445,5 +501,24 @@
       </desc>
     </func>
 
+    <func>
+      <name name="unquote" arity="1" since="OTP 25.0"/>
+      <fsummary>Percent decode characters.</fsummary>
+      <desc>
+        <p>Percent decode characters.</p>
+
+        <p><em>Example:</em></p>
+        <pre>
+1> <input><![CDATA[uri_string:unquote("SomeId%2F04").]]></input>
+<![CDATA["SomeId/04"]]>
+2> <input><![CDATA[uri_string:unquote(<<"SomeId%2F04">>).]]></input>
+<![CDATA[<<"SomeId/04">>]]>
+	</pre>
+
+        <warning><p>Function is not aware about any URI component context and
+        should not be used on whole URI. If applied more than once on the same
+        data, might produce unexpected results.</p></warning>
+      </desc>
+    </func>
   </funcs>
 </erlref>
diff --git a/lib/stdlib/doc/src/uri_string_usage.xml b/lib/stdlib/doc/src/uri_string_usage.xml
index 72851096b7..31e2c23c45 100644
--- a/lib/stdlib/doc/src/uri_string_usage.xml
+++ b/lib/stdlib/doc/src/uri_string_usage.xml
@@ -4,8 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2020</year>
-      <year>2020</year>
+      <year>2022</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -294,12 +293,14 @@
     and it happens when converting a
     <seetype marker="uri_string#uri_map"><c>uri_map()</c></seetype>
     into a <seetype marker="uri_string#uri_string"><c>uri_string()</c></seetype>.
-    There is no equivalent to a raw percent-encoding function as percent-encoding
-    shall be applied on the component level using different sets of allowed characters.
-    Applying percent-encoding directly on an input URI would not be safe just as in
+    Applying any percent-encoding directly on an input URI would not be safe just as in
     the case of
     <seemfa marker="uri_string#percent_decode/1"><c>uri_string:percent_decode/1</c></seemfa>,
     the output could be an invalid URI.
+    Quoting functions allow users to perform raw percent encoding and decoding
+    on application data which cannot be handled automatically by
+    <c>uri_string:recompose/1</c>. For example in scenario when user would
+    need to use '/' or sub-delimeter as data rather than delimeter in a path component.
     </p>
     </note>
   </section>
diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl
index 05b66c12d6..dbbb835036 100644
--- a/lib/stdlib/src/uri_string.erl
+++ b/lib/stdlib/src/uri_string.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2017-2020. All Rights Reserved.
+%% Copyright Ericsson AB 2017-2022. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -237,7 +237,10 @@
          recompose/1,
          resolve/2,
          resolve/3,
-         transcode/2]).
+         transcode/2,
+         quote/1,
+         quote/2,
+         unquote/1]).
 -export_type([error/0,
               uri_map/0,
               uri_string/0]).
@@ -518,6 +521,29 @@ percent_decode(URI) when is_list(URI) orelse
                          is_binary(URI) ->
     raw_decode(URI).
 
+-spec quote(Data) -> QuotedData when
+      Data :: unicode:chardata(),
+      QuotedData :: unicode:chardata().
+quote(D) ->
+    encode(D, fun is_unreserved/1).
+
+-spec quote(Data, Safe) -> QuotedData when
+      Data :: unicode:chardata(),
+      Safe :: string(),
+      QuotedData :: unicode:chardata().
+quote(D, Safe) ->
+    UnreservedOrSafe =
+        fun(C) ->
+                is_unreserved(C) orelse lists:member(C, Safe)
+        end,
+    encode(D, UnreservedOrSafe).
+
+-spec unquote(QuotedData) -> Data when
+      QuotedData :: unicode:chardata(),
+      Data :: unicode:chardata().
+unquote(D) ->
+    raw_decode(D).
+
 %%-------------------------------------------------------------------------
 %% Functions for working with the query part of a URI as a list
 %% of key/value pairs.
diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl
index 04d7a23eab..6a6bff2688 100644
--- a/lib/stdlib/test/uri_string_SUITE.erl
+++ b/lib/stdlib/test/uri_string_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2020. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2022. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -20,6 +20,8 @@
 -module(uri_string_SUITE).
 
 -include_lib("common_test/include/ct.hrl").
+-compile({nowarn_deprecated_function, [{http_uri, encode, 1}]}).
+-compile({nowarn_deprecated_function, [{http_uri, decode, 1}]}).
 
 -export([all/0, suite/0,groups/0,
          normalize/1, normalize_map/1, normalize_return_map/1, normalize_negative/1,
@@ -53,7 +55,8 @@
          interop_query_latin1/1, interop_query_utf8/1,
          regression_parse/1, regression_recompose/1, regression_normalize/1,
          recompose_host_relative_path/1,
-         recompose_host_absolute_path/1
+         recompose_host_absolute_path/1,
+         quote/1
         ]).
 
 
@@ -148,7 +151,8 @@ all() ->
      regression_recompose,
      regression_normalize,
      recompose_host_relative_path,
-     recompose_host_absolute_path
+     recompose_host_absolute_path,
+     quote
     ].
 
 groups() ->
@@ -1354,3 +1358,132 @@ recompose_host_absolute_path(_Config) ->
                                path => [<<"/f">>,<<"oo">>]}),
     ok.
 
+%%-------------------------------------------------------------------------
+%% Quote tests
+%%-------------------------------------------------------------------------
+quote(_Config) ->
+    TestQuote =
+        fun(Unquoted, Quoted) ->
+                Quoted = uri_string:quote(Unquoted)
+        end,
+
+    [TestQuote(U, Q) || #{unquoted := U, quoted := Q} <- get_quote_data()],
+    [TestQuote(U, Q) || #{unquoted_b := U, quoted_b := Q} <- get_quote_data()],
+
+    Head = fun([H | _]) -> H;
+               (<<H, _/binary>>) -> H
+           end,
+
+    TestQuoteUnquote =
+        fun(Unquoted) ->
+                %% case below should be removed when functions used are removed
+                case Head(Unquoted) =< 127 of
+                    true ->
+                        Unquoted = http_uri:decode(http_uri:encode(Unquoted));
+                    _ ->
+                        ok
+                end,
+                Unquoted = uri_string:unquote(uri_string:quote(Unquoted))
+        end,
+    [TestQuoteUnquote(U) || #{unquoted := U} <- get_quote_data()],
+    [TestQuoteUnquote(U) || #{unquoted_b := U} <- get_quote_data()],
+
+    TestQuoteWithSafeList =
+        fun(Unquoted, Quoted) ->
+                Safe = "!$()*", %% characters not encoded by old http_uri:encode
+                Result = uri_string:quote(Unquoted, Safe),
+                %% case below should be removed when function used are removed
+                case Head(Unquoted) =< 127 of
+                    true ->
+                        Result = http_uri:encode(Unquoted);
+                    _ ->
+                        ok
+                end,
+                case lists:member(Head(Unquoted), Safe) of
+                    true ->
+                        Unquoted = Result;
+                    false ->
+                        Quoted = Result
+                end
+        end,
+    [TestQuoteWithSafeList(U, Q) || #{unquoted := U, quoted := Q} <- get_quote_data()],
+    [TestQuoteWithSafeList(U, Q) || #{unquoted_b := U, quoted_b := Q} <- get_quote_data()],
+
+
+    ComposePath = fun (PathSegments, Safe) ->
+                          lists:join("/", [uri_string:quote(S, Safe) ||
+                                              S <- PathSegments])
+                  end,
+
+    %% / used as data see GH-5368
+    ExampleURI1 = "https://internal.api.com/devices/Ethernet0%2F4",
+    ExampleURI1 = uri_string:recompose(
+                    #{scheme => "https",
+                      host => "internal.api.com",
+                      path => ComposePath(["devices", "Ethernet0/4"],
+                                          "")}),
+
+    %% sub-delims as data
+    %% in this example ComposePath must treat sub-delims and '%' as safe character
+    %% to avoid re-encoding encoded characters
+    ExampleURI2 = "yeti://localhost/folder/file.txt,version=1%2C1",
+    ExampleURI2 = uri_string:recompose(
+                    #{scheme => "yeti",
+                      host => "localhost",
+                      path => ComposePath(["folder", "file.txt,version=" ++
+                                           uri_string:quote("1,1")],
+                                          ",=%")}),
+
+    %% percent character as data
+    ExampleURI3 = "yeti://localhost/folder/file_with_%25.txt",
+    ExampleURI3 = uri_string:recompose(
+                    #{scheme => "yeti",
+                      host => "localhost",
+                      path => ComposePath(["folder", "file_with_" ++
+                                               uri_string:quote("%") ++ ".txt"],
+                                          "%")}),
+    ok.
+
+get_quote_data() ->
+    [%% reserved/gen-delims
+     #{unquoted => ":", quoted => "%3A", unquoted_b =><<":">>, quoted_b=> <<"%3A">>},
+     #{unquoted => "/", quoted => "%2F", unquoted_b =><<"/">>, quoted_b=> <<"%2F">>},
+     #{unquoted => "?", quoted => "%3F", unquoted_b =><<"?">>, quoted_b=> <<"%3F">>},
+     #{unquoted => "#", quoted => "%23", unquoted_b =><<"#">>, quoted_b=> <<"%23">>},
+     #{unquoted => "[", quoted => "%5B", unquoted_b =><<"[">>, quoted_b=> <<"%5B">>},
+     #{unquoted => "]", quoted => "%5D", unquoted_b =><<"]">>, quoted_b=> <<"%5D">>},
+     #{unquoted => "@", quoted => "%40", unquoted_b =><<"@">>, quoted_b=> <<"%40">>},
+     %% reserved/sub-delims
+     #{unquoted => "!", quoted => "%21", unquoted_b =><<"!">>, quoted_b=> <<"%21">>},
+     #{unquoted => "$", quoted => "%24", unquoted_b =><<"$">>, quoted_b=> <<"%24">>},
+     #{unquoted => "&", quoted => "%26", unquoted_b =><<"&">>, quoted_b=> <<"%26">>},
+     #{unquoted => "'", quoted => "%27", unquoted_b =><<"'">>, quoted_b=> <<"%27">>},
+     #{unquoted => "(", quoted => "%28", unquoted_b =><<"(">>, quoted_b=> <<"%28">>},
+     #{unquoted => ")", quoted => "%29", unquoted_b =><<")">>, quoted_b=> <<"%29">>},
+     #{unquoted => "*", quoted => "%2A", unquoted_b =><<"*">>, quoted_b=> <<"%2A">>},
+     #{unquoted => "+", quoted => "%2B", unquoted_b =><<"+">>, quoted_b=> <<"%2B">>},
+     #{unquoted => ",", quoted => "%2C", unquoted_b =><<",">>, quoted_b=> <<"%2C">>},
+     #{unquoted => ";", quoted => "%3B", unquoted_b =><<";">>, quoted_b=> <<"%3B">>},
+     #{unquoted => "=", quoted => "%3D", unquoted_b =><<"=">>, quoted_b=> <<"%3D">>},
+     %% other not unreserved
+     #{unquoted => "<", quoted => "%3C", unquoted_b =><<"<">>, quoted_b=> <<"%3C">>},
+     #{unquoted => ">", quoted => "%3E", unquoted_b =><<">">>, quoted_b=> <<"%3E">>},
+     #{unquoted => "\"", quoted => "%22", unquoted_b =><<"\"">>, quoted_b=> <<"%22">>},
+     #{unquoted => "{", quoted => "%7B", unquoted_b =><<"{">>, quoted_b=> <<"%7B">>},
+     #{unquoted => "}", quoted => "%7D", unquoted_b =><<"}">>, quoted_b=> <<"%7D">>},
+     #{unquoted => "|", quoted => "%7C", unquoted_b =><<"|">>, quoted_b=> <<"%7C">>},
+     #{unquoted => "\\", quoted => "%5C", unquoted_b =><<"\\">>, quoted_b=> <<"%5C">>},
+     #{unquoted => "^", quoted => "%5E", unquoted_b =><<"^">>, quoted_b=> <<"%5E">>},
+     #{unquoted => "%", quoted => "%25", unquoted_b =><<"%">>, quoted_b=> <<"%25">>},
+     #{unquoted => " ", quoted => "%20", unquoted_b =><<" ">>, quoted_b=> <<"%20">>},
+     %% non-ASCII
+     #{unquoted => "örebro", quoted => "%C3%B6rebro",
+       unquoted_b =><<"örebro"/utf8>>, quoted_b=> <<"%C3%B6rebro">>},
+     #{unquoted => "Łódź", quoted => "%C5%81%C3%B3d%C5%BA",
+       unquoted_b =><<"Łódź"/utf8>>, quoted_b=> <<"%C5%81%C3%B3d%C5%BA">>},
+     %% unreserved non alpha, non digit characters
+     #{unquoted => "-", quoted => "-", unquoted_b =><<"-">>, quoted_b=> <<"-">>},
+     #{unquoted => ".", quoted => ".", unquoted_b =><<".">>, quoted_b=> <<".">>},
+     #{unquoted => "_", quoted => "_", unquoted_b =><<"_">>, quoted_b=> <<"_">>},
+     #{unquoted => "~", quoted => "~", unquoted_b =><<"~">>, quoted_b=> <<"~">>}
+    ].
-- 
2.34.1

Places

File 3421-stdlib-add-quote-unquote-functions-in-uri_string.patch of Package erlang (Revision 06895323ebe8e4f43b9b3d18d00c12c8)

Places