File 0133-Fix-matching-and-construction-of-utf16-seg... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 0133-Fix-matching-and-construction-of-utf16-segments.patch of Package erlang (Revision 3568a86b92b30ede35d5250ff17067fe)

Currently displaying revision 3568a86b92b30ede35d5250ff17067fe , Show latest

From ab49af0d71e6530e52758fc06363891727e3c680 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 15 Oct 2021 13:09:05 +0200
Subject: [PATCH] Fix matching and construction of utf16 segments

---
 erts/emulator/beam/emu/bs_instrs.tab       |  2 +-
 erts/emulator/beam/emu/ops.tab             | 12 +++++-----
 erts/emulator/beam/generators.tab          | 25 +++++++++++++++++++++
 erts/emulator/beam/jit/x86/generators.tab  | 12 ++++++++++
 erts/emulator/beam/jit/x86/instr_bs.cpp    |  6 ++---
 erts/emulator/beam/jit/x86/ops.tab         |  7 +++---
 erts/emulator/test/bs_construct_SUITE.erl  | 26 ++++++++++++++++++----
 erts/emulator/test/bs_match_misc_SUITE.erl |  4 ++++
 lib/compiler/test/bs_construct_SUITE.erl   | 26 ++++++++++++++++++----
 9 files changed, 100 insertions(+), 20 deletions(-)

diff --git a/erts/emulator/beam/emu/bs_instrs.tab b/erts/emulator/beam/emu/bs_instrs.tab
index c52367de2a..84adc06a9d 100644
--- a/erts/emulator/beam/emu/bs_instrs.tab
+++ b/erts/emulator/beam/emu/bs_instrs.tab
@@ -795,7 +795,7 @@ i_bs_utf16_size(Src, Dst) {
     $Dst = result;
 }
 
-bs_put_utf16(Fail, Flags, Src) {
+i_bs_put_utf16(Fail, Flags, Src) {
     if (!erts_bs_put_utf16(ERL_BITS_ARGS_2($Src, $Flags))) {
         $BADARG($Fail);
     }
diff --git a/erts/emulator/beam/emu/ops.tab b/erts/emulator/beam/emu/ops.tab
index 336c4d0283..e1dba18a9f 100644
--- a/erts/emulator/beam/emu/ops.tab
+++ b/erts/emulator/beam/emu/ops.tab
@@ -1203,8 +1203,8 @@ i_bs_get_utf8 xy f? d
 
 bs_skip_utf8 Fail=f Ms=xy u u => i_bs_get_utf8 Ms Fail x
 
-bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
-bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_get_utf16 Ms Fail Flags x
+bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => get_utf16(Fail, Ms, Flags, Dst)
+bs_skip_utf16 Fail=f Ms=xy Unit=u Flags=u => bs_get_utf16 Fail Ms Unit Flags x
 
 i_bs_get_utf16 xy f? t d
 
@@ -1303,6 +1303,8 @@ bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst
 
 bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src
 
+bs_put_utf16 Fail Flags Src => put_utf16(Fail, Flags, Src)
+
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
 
@@ -1310,15 +1312,15 @@ i_bs_utf8_size S x
 i_bs_utf16_size S x
 
 i_bs_put_utf8 j? S
-bs_put_utf16 j? t S
+i_bs_put_utf16 j? t S
 
 i_bs_validate_unicode j? S
 
-# Handle unoptimized code.
+# Handle unoptimized code and the 'native' flag for utf16 segments.
 i_bs_utf8_size Src=c Dst => move Src x | i_bs_utf8_size x Dst
 i_bs_utf16_size Src=c Dst => move Src x | i_bs_utf16_size x Dst
 i_bs_put_utf8 Fail Src=c => move Src x | i_bs_put_utf8 Fail x
-bs_put_utf16 Fail Flags Src=c => move Src x | bs_put_utf16 Fail Flags x
+i_bs_put_utf16 Fail Flags Src=c => move Src x | i_bs_put_utf16 Fail Flags x
 i_bs_validate_unicode Fail Src=c => move Src x | i_bs_validate_unicode Fail x
 
 #
diff --git a/erts/emulator/beam/generators.tab b/erts/emulator/beam/generators.tab
index 520c688113..9c9d357475 100644
--- a/erts/emulator/beam/generators.tab
+++ b/erts/emulator/beam/generators.tab
@@ -93,6 +93,19 @@ gen.get_float2(Fail, Ms, Live, Size, Unit, Flags, Dst) {
     return op;
 }
 
+gen.get_utf16(Fail, Ms, Flags, Dst) {
+    BeamOp* op;
+    $NewBeamOp(S, op);
+
+    $NativeEndian(Flags);
+    $BeamOpNameArity(op, i_bs_get_utf16, 4);
+    op->a[0] = Ms;
+    op->a[1] = Fail;
+    op->a[2] = Flags;
+    op->a[3] = Dst;
+    return op;
+}
+
 gen.put_binary(Fail, Size, Unit, Flags, Src) {
     BeamOp* op;
     $NewBeamOp(S, op);
@@ -234,6 +247,18 @@ gen.put_float(Fail, Size, Unit, Flags, Src) {
     return op;
 }
 
+gen.put_utf16(Fail, Flags, Src) {
+    BeamOp* op;
+    $NewBeamOp(S, op);
+
+    $NativeEndian(Flags);
+    $BeamOpNameArity(op, i_bs_put_utf16, 3);
+    op->a[0] = Fail;
+    op->a[1] = Flags;
+    op->a[2] = Src;
+    return op;
+}
+
 // Generate the fastest instruction for bs_skip_bits.
 gen.skip_bits2(Fail, Ms, Size, Unit, Flags) {
     BeamOp* op;
diff --git a/erts/emulator/beam/jit/x86/generators.tab b/erts/emulator/beam/jit/x86/generators.tab
index ab755fde4c..e09c9b5c7b 100644
--- a/erts/emulator/beam/jit/x86/generators.tab
+++ b/erts/emulator/beam/jit/x86/generators.tab
@@ -451,6 +451,18 @@ gen.get_binary2(Fail, Ms, Live, Size, Unit, Flags, Dst) {
     return op;
 }
 
+gen.skip_utf16(Fail, Ms, Flags) {
+    BeamOp* op;
+    $NewBeamOp(S, op);
+
+    $NativeEndian(Flags);
+    $BeamOpNameArity(op, i_bs_skip_utf16, 3);
+    op->a[0] = Ms;
+    op->a[1] = Fail;
+    op->a[2] = Flags;
+    return op;
+}
+
 gen.combine_conses(Len, Dst, Hd) {
     BeamOp* cons;
     BeamOp* tail;
diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp
index 7df0b84cae..508e688100 100644
--- a/erts/emulator/beam/jit/x86/instr_bs.cpp
+++ b/erts/emulator/beam/jit/x86/instr_bs.cpp
@@ -1224,9 +1224,9 @@ void BeamModuleAssembler::emit_i_bs_utf16_size(const ArgVal &Src,
     mov_arg(Dst, RET);
 }
 
-void BeamModuleAssembler::emit_bs_put_utf16(const ArgVal &Fail,
-                                            const ArgVal &Flags,
-                                            const ArgVal &Src) {
+void BeamModuleAssembler::emit_i_bs_put_utf16(const ArgVal &Fail,
+                                              const ArgVal &Flags,
+                                              const ArgVal &Src) {
     Label next;
 
     if (Fail.getValue() == 0) {
diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab
index d6af3818a1..8ccfb6cfbc 100644
--- a/erts/emulator/beam/jit/x86/ops.tab
+++ b/erts/emulator/beam/jit/x86/ops.tab
@@ -886,8 +886,8 @@ i_bs_get_utf8 S f? d
 bs_skip_utf8 Fail=f Ms=xy u u => i_bs_skip_utf8 Ms Fail
 i_bs_skip_utf8 S f?
 
-bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
-bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_skip_utf16 Ms Fail Flags
+bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => get_utf16(Fail, Ms, Flags, Dst)
+bs_skip_utf16 Fail=f Ms=xy u Flags=u => skip_utf16(Fail, Ms, Flags)
 
 i_bs_get_utf16 S f? t d
 i_bs_skip_utf16 S f? t
@@ -983,6 +983,7 @@ bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst
 bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst
 
 bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src
+bs_put_utf16 Fail Flags Src => put_utf16(Fail, Flags, Src)
 
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
@@ -991,7 +992,7 @@ i_bs_utf8_size s x
 i_bs_utf16_size s x
 
 i_bs_put_utf8 j? s
-bs_put_utf16 j? t s
+i_bs_put_utf16 j? t s
 
 i_bs_validate_unicode j? s
 
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 81107c5644..a30c394971 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -132,6 +132,12 @@ l(I_13, I_big1) ->
      ?T(<<32978297842987249827298387697777669766334937:128/native-integer>>,
 	native_bignum()),
 
+     ?T(<<$э/native-utf16,$т/native-utf16,$о/native-utf16," спутник"/native-utf16>>,
+        native_utf16()),
+
+     ?T(<<$в/native-utf32,"ода"/native-utf32>>,
+	native_utf32()),
+
      %% Unit tests.
      ?T(<<<<5:3>>/bitstring>>, <<5:3>>),
      ?T(<<42,<<7:4>>/binary-unit:4>>, <<42,7:4>>),
@@ -152,6 +158,18 @@ native_bignum() ->
 	<<1,0>> -> [217,73,11,176,128,109,231,39,101,170,213,1,177,18,205,129]
     end.
 
+native_utf16() ->
+    case <<1:16/native>> of
+	<<0,1>> -> [4,77,4,66,4,62,0,32,4,65,4,63,4,67,4,66,4,61,4,56,4,58];
+        <<1,0>> -> [77,4,66,4,62,4,32,0,65,4,63,4,67,4,66,4,61,4,56,4,58,4]
+    end.
+
+native_utf32() ->
+    case <<1:16/native>> of
+	<<0,1>> -> [0,0,4,50,0,0,4,62,0,0,4,52,0,0,4,48];
+        <<1,0>> -> [50,4,0,0,62,4,0,0,52,4,0,0,48,4,0,0]
+    end.
+
 evaluate(Str, Vars) ->
     {ok,Tokens,_} =
 	erl_scan:string(Str ++ " . "),
@@ -173,13 +191,13 @@ eval_list([{C_bin, Str, Bytes} | Rest], Vars) ->
     end.
 
 one_test({C_bin, E_bin, Str, Bytes}) when is_list(Bytes) ->
-    io:format("  ~s, ~p~n", [Str, Bytes]),
+    io:format("  ~ts, ~p~n", [Str, Bytes]),
     Bin = list_to_binary(Bytes),
     if
 	C_bin == Bin ->
 	    ok;
 	true ->
-	    io:format("ERROR: Compiled: ~p. Expected ~p. Got ~p.~n",
+	    io:format("ERROR: Compiled: ~p.~nExpected ~p.~nGot ~p.~n",
 		      [Str, Bytes, binary_to_list(C_bin)]),
 	    ct:fail(comp)
     end,
@@ -187,12 +205,12 @@ one_test({C_bin, E_bin, Str, Bytes}) when is_list(Bytes) ->
 	E_bin == Bin ->
 	    ok;
 	true ->
-	    io:format("ERROR: Interpreted: ~p. Expected ~p. Got ~p.~n",
+	    io:format("ERROR: Interpreted: ~p.~nExpected ~p.~nGot ~p.~n",
 		      [Str, Bytes, binary_to_list(E_bin)]),
 	    ct:fail(comp)
     end;
 one_test({C_bin, E_bin, Str, Result}) ->
-    io:format("  ~s ~p~n", [Str, C_bin]),
+    io:format("  ~ts ~p~n", [Str, C_bin]),
     if
 	C_bin == E_bin ->
 	    ok;
diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl
index 42992d2f46..bd10284c49 100644
--- a/erts/emulator/test/bs_match_misc_SUITE.erl
+++ b/erts/emulator/test/bs_match_misc_SUITE.erl
@@ -291,11 +291,15 @@ native(Config) when is_list(Config) ->
 native_big() ->
     <<37.33:64/native-float>> = <<37.33:64/big-float>>,
     <<3974:16/native-integer>> = <<3974:16/big-integer>>,
+    <<$W/native-utf16>> = id(<<0,$W>>),
+    <<$W/native-utf32>> = id(<<0,0,0,$W>>),
     {comment,"Big endian"}.
 
 native_little() ->
     <<37869.32343:64/native-float>> = <<37869.32343:64/little-float>>,
     <<7974:16/native-integer>> = <<7974:16/little-integer>>,
+    <<$W/native-utf16>> = id(<<$W,0>>),
+    <<$W/native-utf32>> = id(<<$W,0,0,0>>),
     {comment,"Little endian"}.
 
 happi(Config) when is_list(Config) ->
diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl
index a5f59e8bc3..b314964deb 100644
--- a/lib/compiler/test/bs_construct_SUITE.erl
+++ b/lib/compiler/test/bs_construct_SUITE.erl
@@ -140,10 +140,16 @@ l(I_13, I_big1, I_16, Bin) ->
 	[129]),
      ?T(<<1:3,"string",9:5>>,
 	[46,110,142,77,45,204,233]),
+
+     %% Test the native flag.
      ?T(<<37.98:64/native-float>>,
 	native_3798()),
      ?T(<<32978297842987249827298387697777669766334937:128/native-integer>>,
 	native_bignum()),
+     ?T(<<$э/native-utf16,$т/native-utf16,$о/native-utf16," спутник"/native-utf16>>,
+        native_utf16()),
+     ?T(<<$в/native-utf32,"ода"/native-utf32>>,
+	native_utf32()),
 
      ?T(<<Bin/binary>>,
         [165,90,195]),
@@ -189,6 +195,18 @@ native_bignum() ->
 	<<1,0>> -> [217,73,11,176,128,109,231,39,101,170,213,1,177,18,205,129]
     end.
 
+native_utf16() ->
+    case <<1:16/native>> of
+	<<0,1>> -> [4,77,4,66,4,62,0,32,4,65,4,63,4,67,4,66,4,61,4,56,4,58];
+        <<1,0>> -> [77,4,66,4,62,4,32,0,65,4,63,4,67,4,66,4,61,4,56,4,58,4]
+    end.
+
+native_utf32() ->
+    case <<1:16/native>> of
+	<<0,1>> -> [0,0,4,50,0,0,4,62,0,0,4,52,0,0,4,48];
+        <<1,0>> -> [50,4,0,0,62,4,0,0,52,4,0,0,48,4,0,0]
+    end.
+
 evaluate(Str, Vars) ->
     {ok,Tokens,_} =
 	erl_scan:string(Str ++ " . "),
@@ -210,13 +228,13 @@ eval_list([{C_bin, Str, Bytes} | Rest], Vars) ->
     end.
 
 one_test({C_bin, E_bin, Str, Bytes}) when is_list(Bytes) ->
-    io:format("  ~s, ~p~n", [Str, Bytes]),
+    io:format("  ~ts, ~p~n", [Str, Bytes]),
     Bin = list_to_bitstring(Bytes),
     if
 	C_bin == Bin ->
 	    ok;
 	true ->
-	    io:format("ERROR: Compiled: ~p. Expected ~p. Got ~p.~n",
+	    io:format("ERROR: Compiled: ~p.~nExpected ~p.~nGot ~p.~n",
 		      [Str, Bytes, bitstring_to_list(C_bin)]),
 	    ct:fail(comp)
     end,
@@ -224,12 +242,12 @@ one_test({C_bin, E_bin, Str, Bytes}) when is_list(Bytes) ->
 	E_bin == Bin ->
 	    ok;
 	true ->
-	    io:format("ERROR: Interpreted: ~p. Expected ~p. Got ~p.~n",
+	    io:format("ERROR: Interpreted: ~p.~nExpected ~p.~nGot ~p.~n",
 		      [Str, Bytes, bitstring_to_list(E_bin)]),
 	    ct:fail(comp)
     end;
 one_test({C_bin, E_bin, Str, Result}) ->
-    io:format("  ~s ~p~n", [Str, C_bin]),
+    io:format("  ~ts ~p~n", [Str, C_bin]),
     if
 	C_bin == E_bin ->
 	    ok;
-- 
2.31.1

Places

File 0133-Fix-matching-and-construction-of-utf16-segments.patch of Package erlang (Revision 3568a86b92b30ede35d5250ff17067fe)

Places