File 0277-Ensure-reproducible-builds-of-Erlang-code.... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 0277-Ensure-reproducible-builds-of-Erlang-code.patch of Package erlang (Revision ab414368c0f53cdf471f751b019c6c4f)

Currently displaying revision ab414368c0f53cdf471f751b019c6c4f , Show latest

From 02e6b25ce497fbf629f01a9bca9df5247b2f9039 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 23 Aug 2021 13:36:48 +0200
Subject: [PATCH] Ensure reproducible builds of Erlang code

The `deterministic` option for the compiler is not always sufficient
to make builds reproducible. There remains the problems with map
literals having more than 32 elements. Literals are converted to the
external term format by term_to_binary/2 before embedding them in the
BEAM file. However, the order in which individual map elements are
encoded for a map with more than 32 elements can vary.

To fix this issue, implement the option `deterministic` for
term_to_binary/2. When `deterministic` is given, term_to_binary/2
sorts map elements into a total term order before encoding them.

Also update the compiler to use `deterministic` when encoding
literals, and also to stop evaluating term_to_binary/1 with a literal
argument.
---
 erts/doc/src/erlang.xml             |   5 +
 erts/emulator/beam/atom.names       |   1 +
 erts/emulator/beam/dist.h           |  14 ++-
 erts/emulator/beam/erl_alloc.types  |   2 +
 erts/emulator/beam/external.c       | 168 +++++++++++++++++++++++++++-
 erts/emulator/test/binary_SUITE.erl | 164 ++++++++++++++++++++++++++-
 erts/emulator/test/map_SUITE.erl    |   4 +-
 erts/preloaded/ebin/erlang.beam     | Bin 131424 -> 131428 bytes
 erts/preloaded/src/erlang.erl       |   2 +
 lib/compiler/src/beam_dict.erl      |   2 +-
 lib/compiler/src/erl_bifs.erl       |   1 -
 11 files changed, 354 insertions(+), 9 deletions(-)

diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 4367f6f757..331bd0ad82 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -11110,6 +11110,11 @@ hello
 	 systems as of R16B can decode this representation.
 	 </item>
 	 </taglist>
+ Option <c>deterministic</c> (introduced in OTP 24.1) can
+ be used to ensure that within the same major release of Erlang/OTP,
+ the same encoded representation is returned for the same
+ term. There is still no guarantee that the encoded representation
+ remains the same between major releases of Erlang/OTP.
 See also <seemfa marker="#binary_to_term/1">
 <c>binary_to_term/1</c></seemfa>.
 </desc>
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 0d2abbc57e..9b1d2c4196 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -210,6 +210,7 @@ atom decimals
 atom default
 atom delay_trap
 atom demonitor
+atom deterministic
 atom dictionary
 atom dirty_bif_exception
 atom dirty_bif_result
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index b5c8d4d5f9..12392955e6 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -62,6 +62,13 @@
 #define DFLAG_V4_NC (((Uint64)0x4) << 32)
 #define DFLAG_ALIAS (((Uint64)0x8) << 32)
 
+/*
+ * In term_to_binary/2, we will use DFLAG_ATOM_CACHE to mean
+ * DFLAG_DETERMINISTIC.
+ */
+
+#define DFLAG_DETERMINISTIC DFLAG_ATOM_CACHE
+
 /* Mandatory flags for distribution */
 #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \
 | DFLAG_EXTENDED_PIDS_PORTS \
@@ -262,6 +269,9 @@ typedef struct TTBEncodeContext_ {
 byte* ep;
 Eterm obj;
 ErtsWStack wstack;
+ Eterm* map_array;
+ Eterm* next_map_element;
+ void* ycf_yield_state;
 Binary *result_bin;
 byte *cptr;
 Sint vlen;
@@ -284,8 +294,10 @@ typedef struct TTBEncodeContext_ {
 #define ERTS_INIT_TTBEncodeContext(Ctx, Flags) \
 do { \
 (Ctx)->wstack.wstart = NULL; \
- (Ctx)->dflags = (Flags); \
+ (Ctx)->dflags = (Flags); \
 (Ctx)->level = 0; \
+ (Ctx)->map_array = 0; \
+ (Ctx)->ycf_yield_state = 0; \
 (Ctx)->vlen = 0; \
 (Ctx)->size = 0; \
 (Ctx)->termv = NULL; \
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 89f5f8e009..bb6d00ab8c 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -302,6 +302,8 @@ type	FP_EXCEPTION	LONG_LIVED	SYSTEM		fp_exception
 type	LL_MPATHS	LONG_LIVED	SYSTEM		ll_migration_paths
 type	SL_MPATHS	SHORT_LIVED	SYSTEM		sl_migration_paths
 
+type	T2B_DETERMINISTIC SHORT_LIVED	PROCESSES	term_to_binary_deterministic
+
 #
 # Types used for special emulators
 #
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f6bb7f3816..81308db12c 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1426,6 +1426,7 @@ parse_t2b_opts(Eterm opts, Uint *flagsp, int *levelp, int *iovecp, Uint *fsizep)
 int level = 0;
 int iovec = 0;
 Uint flags = TERM_TO_BINARY_DFLAGS;
+ int deterministic = 0;
 Uint fsize = ~((Uint) 0); /* one fragment */
 
 while (is_list(opts)) {
@@ -1436,6 +1437,8 @@ parse_t2b_opts(Eterm opts, Uint *flagsp, int *levelp, int *iovecp, Uint *fsizep)
 }
 else if (iovecp && arg == am_iovec) {
 iovec = !0;
+ } else if (arg == am_deterministic) {
+ deterministic = 1;
 	} else if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) {
 	 if (tp[1] == am_minor_version && is_small(tp[2])) {
 		switch (signed_val(tp[2])) {
@@ -1477,6 +1480,10 @@ parse_t2b_opts(Eterm opts, Uint *flagsp, int *levelp, int *iovecp, Uint *fsizep)
 return 0; /* badarg */
 }
 
+ if (deterministic) {
+ flags |= DFLAG_DETERMINISTIC;
+ }
+
 *flagsp = flags;
 *levelp = level;
 if (iovecp)
@@ -2296,6 +2303,10 @@ static int ttb_context_destructor(Binary *context_bin)
 		erts_bin_free(context->s.ec.result_bin);
 		context->s.ec.result_bin = NULL;
 	 }
+ if (context->s.ec.map_array)
+ erts_free(ERTS_ALC_T_T2B_DETERMINISTIC, context->s.ec.map_array);
+ if (context->s.ec.ycf_yield_state)
+ erts_qsort_ycf_gen_destroy(context->s.ec.ycf_yield_state);
 if (context->s.ec.iov)
 erts_free(ERTS_ALC_T_T2B_VEC, context->s.ec.iov);
 	 break;
@@ -3024,7 +3035,43 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, const byte* ep,
 #define ENC_BIN_COPY ((Eterm) 3)
 #define ENC_MAP_PAIR ((Eterm) 4)
 #define ENC_HASHMAP_NODE ((Eterm) 5)
-#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6)
+#define ENC_STORE_MAP_ELEMENT ((Eterm) 6)
+#define ENC_START_SORTING_MAP ((Eterm) 7)
+#define ENC_CONTINUE_SORTING_MAP ((Eterm) 8)
+#define ENC_PUSH_SORTED_MAP ((Eterm) 9)
+#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 10)
+
+static Eterm* alloc_map_array(Uint size)
+{
+ return (Eterm *) erts_alloc(ERTS_ALC_T_T2B_DETERMINISTIC,
+ size * 2 * sizeof(Eterm));
+}
+
+static int map_key_compare(Eterm *a, Eterm *b)
+{
+ Sint c = CMP_TERM(*a, *b);
+ if (c < 0) {
+ return -1;
+ } else if (c > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+static void*
+ycf_yield_alloc(size_t size, void* context)
+{
+ (void) context;
+ return (void *) erts_alloc(ERTS_ALC_T_T2B_DETERMINISTIC, size);
+}
+
+static void
+ycf_yield_free(void* block, void* context)
+{
+ (void) context;
+ erts_free(ERTS_ALC_T_T2B_DETERMINISTIC, block);
+}
 
 static byte*
 enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint64 dflags,
@@ -3047,9 +3094,14 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 Uint* ptr;
 Eterm val;
 FloatDef f;
- Sint r = 0;
+ register Sint r = 0;
 int use_iov = 0;
 
+ /* The following variables are only used during encoding of
+ * a map when the `deterministic` option is active. */
+ Eterm* map_array = NULL;
+ Eterm* next_map_element = NULL;
+
 if (ctx) {
 	WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
@@ -3059,6 +3111,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	 WSTACK_RESTORE(s, &ctx->wstack);
 	 ep = ctx->ep;
 	 obj = ctx->obj;
+ map_array = ctx->map_array;
+ next_map_element = ctx->next_map_element;
 	 if (is_non_value(obj)) {
 		goto outer_loop;
 	 }
@@ -3143,6 +3197,88 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		obj = CAR(ptr);
 	 }
 	 break;
+	case ENC_STORE_MAP_ELEMENT: /* option `deterministic` */
+	 if (is_list(obj)) { /* leaf node [K|V] */
+		ptr = list_val(obj);
+ *next_map_element++ = CAR(ptr);
+ *next_map_element++ = CDR(ptr);
+ goto outer_loop;
+	 }
+	 break;
+	case ENC_START_SORTING_MAP: /* option `deterministic` */
+ {
+ long num_reductions = r;
+
+ n = next_map_element - map_array;
+ ASSERT(n > MAP_SMALL_MAP_LIMIT);
+ if (ctx == NULL) {
+ /* No context means that the external representation of term
+ * being encoded will fit in a heap binary (64 bytes). This
+ * can only happen in the DEBUG build of the runtime system
+ * where maps with more than 3 elements are large maps. */
+ ASSERT(n < 64); /* Conservative assertion. */
+ qsort(map_array, n/2, 2*sizeof(Eterm),
+ (int (*)(const void *, const void *)) map_key_compare);
+ WSTACK_PUSH2(s, ENC_PUSH_SORTED_MAP, THE_NON_VALUE);
+ goto outer_loop;
+ } else {
+ /* Use yieldable qsort since the number of elements
+ * in the map could be huge. */
+ num_reductions = r;
+ ctx->ycf_yield_state = NULL;
+ erts_qsort_ycf_gen_yielding(&num_reductions,
+ &ctx->ycf_yield_state,
+ NULL,
+ ycf_yield_alloc,
+ ycf_yield_free,
+ NULL,
+ 0,
+ NULL,
+ map_array, n/2, 2*sizeof(Eterm),
+ (int (*)(const void *, const void *)) map_key_compare);
+ if (ctx->ycf_yield_state) {
+ r = 0;
+ WSTACK_PUSH2(s, ENC_CONTINUE_SORTING_MAP, THE_NON_VALUE);
+ break;
+ } else {
+ WSTACK_PUSH2(s, ENC_PUSH_SORTED_MAP, THE_NON_VALUE);
+ r = num_reductions;
+ goto outer_loop;
+ }
+ }
+ }
+ case ENC_CONTINUE_SORTING_MAP: /* option `deterministic` */
+ {
+ long num_reductions = r;
+
+ erts_qsort_ycf_gen_continue(&num_reductions,
+ &ctx->ycf_yield_state,
+ NULL);
+ if (ctx->ycf_yield_state) {
+ r = 0;
+ WSTACK_PUSH2(s, ENC_CONTINUE_SORTING_MAP, THE_NON_VALUE);
+ break;
+ } else {
+ WSTACK_PUSH2(s, ENC_PUSH_SORTED_MAP, THE_NON_VALUE);
+ r = num_reductions;
+ goto outer_loop;
+ }
+ }
+ case ENC_PUSH_SORTED_MAP: /* option `deterministic` */
+ {
+ n = next_map_element - map_array;
+ WSTACK_RESERVE(s, 2*n);
+ ptr = next_map_element - 1;
+ do {
+ WSTACK_FAST_PUSH(s, ENC_TERM);
+ WSTACK_FAST_PUSH(s, *ptr);
+ ptr--;
+ } while (ptr > map_array);
+ obj = *ptr;
+ erts_free(ERTS_ALC_T_T2B_DETERMINISTIC, map_array);
+ map_array = next_map_element = NULL;
+ break;
+ }
 	case ENC_LAST_ARRAY_ELEMENT:
 	 /* obj is the tuple */
 	 {
@@ -3163,6 +3299,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	 *reds = 0;
 	 ctx->obj = obj;
 	 ctx->ep = ep;
+ ctx->map_array = map_array;
+ ctx->next_map_element = next_map_element;
 	 WSTACK_SAVE(s, &ctx->wstack);
 	 return -1;
 	}
@@ -3350,20 +3488,42 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	 } else {
 		Eterm hdr;
 		Uint node_sz;
+ Eterm node_processor;
 		ptr = boxed_val(obj);
 		hdr = *ptr;
 		ASSERT(is_header(hdr));
+
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 		case HAMT_SUBTAG_HEAD_ARRAY:
 		 *ep++ = MAP_EXT;
 		 ptr++;
 		 put_int32(*ptr, ep); ep += 4;
+ if (dflags & DFLAG_DETERMINISTIC) {
+ /* Option `deterministic`: Note that we
+ * process large maps in a breadth-first
+ * order, that is, we push all keys and values
+ * to the stack and deallocate the map array
+ * before encoding any of the keys and
+ * values. That means that when we find a
+ * large map in key or value of an outer map,
+ * the map array for the outer map has already
+ * been deallocated. */
+
+ ASSERT(map_array == NULL);
+ next_map_element = map_array = alloc_map_array(*ptr);
+ WSTACK_PUSH2(s, ENC_START_SORTING_MAP, THE_NON_VALUE);
+ }
 		 node_sz = 16;
 		 break;
 		case HAMT_SUBTAG_HEAD_BITMAP:
 		 *ep++ = MAP_EXT;
 		 ptr++;
 		 put_int32(*ptr, ep); ep += 4;
+ if (dflags & DFLAG_DETERMINISTIC) {
+ ASSERT(map_array == NULL);
+ next_map_element = map_array = alloc_map_array(*ptr);
+ WSTACK_PUSH2(s, ENC_START_SORTING_MAP, THE_NON_VALUE);
+ }
 		 /*fall through*/
 		case HAMT_SUBTAG_NODE_BITMAP:
 		 node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
@@ -3374,9 +3534,11 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		}
 
 		ptr++;
+ node_processor = (dflags & DFLAG_DETERMINISTIC) ?
+ ENC_STORE_MAP_ELEMENT : ENC_HASHMAP_NODE;
 		WSTACK_RESERVE(s, node_sz*2);
 		while(node_sz--) {
-		 WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE);
+ WSTACK_FAST_PUSH(s, node_processor);
 		 WSTACK_FAST_PUSH(s, *ptr++);
 		}
 	 }
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index d8a4e6cddf..2840eb3dd0 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -49,7 +49,7 @@
 	 bad_list_to_binary/1, bad_binary_to_list/1,
 	 t_split_binary/1, bad_split/1,
 	 terms/1, terms_float/1, float_middle_endian/1,
- b2t_used_big/1,
+ b2t_used_big/1, t2b_deterministic/1,
 	 external_size/1, t_iolist_size/1,
 t_iolist_size_huge_list/1,
 t_iolist_size_huge_bad_arg_list/1,
@@ -90,7 +90,7 @@ all() ->
 t_iolist_size_huge_list,
 t_iolist_size_huge_bad_arg_list,
 {group, iolist_size_benchmarks},
- b2t_used_big,
+ b2t_used_big, t2b_deterministic,
 bad_binary_to_term_2, safe_binary_to_term2,
 bad_binary_to_term, bad_terms, t_hash, bad_size,
 sub_bin_copy, bad_term_to_binary, t2b_system_limit,
@@ -579,18 +579,28 @@ terms(Config) when is_list(Config) ->
 TestFun = fun(Term) ->
 S = io_lib:format("~p", [Term]),
 io:put_chars(S),
+
 		 Bin = term_to_binary(Term),
 		 case erlang:external_size(Bin) of
 			 Sz when is_integer(Sz), size(Bin) =< Sz ->
 			 ok
 		 end,
+
 Bin1 = term_to_binary(Term, [{minor_version, 1}]),
 case erlang:external_size(Bin1, [{minor_version, 1}]) of
 Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 ->
 ok
 end,
+
+ BinDet = term_to_binary(Term, [deterministic]),
+ case erlang:external_size(Bin) of
+ DetSz when is_integer(DetSz), byte_size(Bin) =< DetSz ->
+ ok
+ end,
+
 		 Term = binary_to_term_stress(Bin),
 		 Term = binary_to_term_stress(Bin, [safe]),
+		 Term = binary_to_term_stress(BinDet),
 Bin_sz = byte_size(Bin),
 		 {Term,Bin_sz} = binary_to_term_stress(Bin, [used]),
 
@@ -676,6 +686,156 @@ float_middle_endian(Config) when is_list(Config) ->
 <<131,70,63,240,0,0,0,0,0,0>> = term_to_binary(1.0, [{minor_version,1}]),
 1.0 = binary_to_term_stress(<<131,70,63,240,0,0,0,0,0,0>>).
 
+%% Test term_to_binary(Term, [deterministic]).
+t2b_deterministic(_Config) ->
+ _ = rand:uniform(),				%Seed generator
+ io:format("Seed: ~p", [rand:export_seed()]),
+ Map0 = t2b_deterministic_1(100, #{1 => a, 1.0 => b}),
+
+ Map1 = maps:merge(make_map(10000), Map0),
+ test_deterministic(Map1),
+
+ case total_memory() of
+ Amount when Amount > 15 ->
+ t2b_deterministic_heavy(Map0);
+ _ ->
+ {comment,"heavy tests skipped"}
+ end.
+
+t2b_deterministic_heavy(Map0) ->
+ Map1 = maps:merge(make_map(1_000_000), Map0),
+ test_deterministic(Map1),
+
+ %% If the external representation of the term to be encoded is
+ %% estimated to fit in a heap binary (64 bytes), the internal
+ %% enc_term_int() function that does the actual encoding will be
+ %% called without an context and thus is unable to trap. In the
+ %% DEBUG build of the runtime system, maps with more than 3
+ %% elements will be large maps, and depending on the keys and
+ %% values, it is possible that a large map can fit in 64
+ %% bytes. Therefore, the encoding code for deterministic maps must
+ %% cannot assume that there always exists a context.
+
+ lists:foreach(fun(N) ->
+ Map = maps:from_list([{E,E} || E <- lists:seq(1, N)]),
+ test_deterministic(Map)
+ end, lists:seq(4, 16)),
+
+ %% Test that terminating a process that is encoding a huge map
+ %% will neither crash the runtime system nor leak memory. (Run the
+ %% test with memory sanitizer or valgrind to catch potential memory
+ %% leaks.)
+ erlang:trace_pattern({erlang, term_to_binary, 2}, true, []),
+ Map = huge_map(1_000_000, []),
+ rinse_and_repeat(2, Map).
+
+rinse_and_repeat(Wait, Map) ->
+ {Pid,Ref} = spawn_monitor(fun() ->
+ receive go -> ok end,
+ term_to_binary(Map, [deterministic])
+ end),
+ erlang:trace(Pid, true, [call,arity]),
+
+ Pid ! go,
+ receive
+ {trace,Pid,call,{erlang,term_to_binary,2}} ->
+ %% Wait a short period to avoid killing term_to_binary/2 during
+ %% size calculation part.
+ receive after Wait -> ok end,
+ exit(Pid, kill);
+ Other ->
+ error(Other)
+ end,
+
+ receive
+ {'DOWN',Ref,process,Pid,Reason} ->
+ case Reason of
+ normal ->
+ %% The encoding operation finished before the process
+ %% was forcefully killed. Done.
+ ok;
+ killed ->
+ %% The process was killed in term_to_binary/2. We
+ %% can't know whether it was killed too early, so
+ %% we will try again with a slightly longer time.
+ rinse_and_repeat(Wait + Wait div 2, Map)
+ end;
+ Other2 ->
+ error(Other2)
+ end.
+
+huge_map(0, Acc) ->
+ maps:from_list(Acc);
+huge_map(N, Acc) ->
+ %% Create awkward keys to slow down the sorting.
+ Key = <<N:127/big>>,
+ huge_map(N-1, [{Key,N}|Acc]).
+
+t2b_deterministic_1(0, Map) ->
+ Map;
+t2b_deterministic_1(N, Map0) ->
+ test_deterministic(Map0),
+ Map = Map0#{random_term() => N},
+ t2b_deterministic_1(N - 1, Map).
+
+test_deterministic(Map) ->
+ Bin = term_to_binary(Map, [deterministic]),
+ Map = binary_to_term(Bin),
+
+ %% Make sure that the keys for the map are ordered.
+ List0 = maps:to_list(Map),
+ List = lists:sort(fun(A, B) -> erts_internal:cmp_term(A, B) =< 0 end, List0),
+ List = decode_ext_map(Bin),
+ ok.
+
+decode_ext_map(MapBin) ->
+ MapTag = 116,
+ ListTag = 108,
+ NilTag = 106,
+
+ %% Rewrite the map to a list before decoding to preserve the order
+ %% the map elements.
+ <<131,MapTag,Size:32,Bin0/binary>> = MapBin,
+ ListBin = <<131,ListTag,(2*Size):32,Bin0/binary,NilTag>>,
+ List = binary_to_term(ListBin),
+ decode_ext_map_1(List).
+
+decode_ext_map_1([Key,Val|T]) ->
+ [{Key,Val}|decode_ext_map_1(T)];
+decode_ext_map_1([]) -> [].
+
+make_map(N) ->
+ maps:from_list([{I,I*I} || I <- lists:seq(1, N)]).
+
+random_term() ->
+ case rand:uniform(11) of
+ 1 ->
+ list_to_atom(integer_to_list(36#aaaa + rand:uniform(100) * 23, 36));
+ 2 ->
+ rand:uniform();
+ 3 ->
+ rand:uniform(10000000);
+ 4 ->
+ (1 bsl rand:uniform(128)) + rand:uniform(10000000);
+ 5 ->
+ self();
+ 6 ->
+ make_ref();
+ 7 ->
+ random_list();
+ 8 ->
+ list_to_tuple(random_list());
+ 9 ->
+ [];
+ 10 ->
+ make_map(33);
+ 11 ->
+ make_map(rand:uniform(10))
+ end.
+
+random_list() ->
+ [random_term() || _ <- lists:seq(1, rand:uniform(10)-1)].
+
 external_size(Config) when is_list(Config) ->
 %% Build a term whose external size only fits in a big num (on 32-bit CPU).
 external_size_1(16#11111111111111117777777777777777888889999, 0, 16#FFFFFFF),
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index c6302d0239..5707a8e56c 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -2346,7 +2346,8 @@ t_map_encode_decode(Config) when is_list(Config) ->
 			M1 = M0#{Key => Key},
 			case Key rem 17 of
 			 0 ->
-				M1 = binary_to_term(term_to_binary(M1));
+				M1 = binary_to_term(term_to_binary(M1)),
+ M1 = binary_to_term(term_to_binary(M1, [deterministic]));
 			 _ ->
 				ok
 			end,
@@ -2884,6 +2885,7 @@ t_erts_internal_order(_Config) when is_list(_Config) ->
 
 M = #{0 => 0,2147483648 => 0},
 true = M =:= binary_to_term(term_to_binary(M)),
+ true = M =:= binary_to_term(term_to_binary(M, [deterministic])),
 
 F1 = fun(_, _) -> 0 end,
 F2 = fun(_, _) -> 1 end,
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index b9196448fe..0c43b690a0 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -2679,6 +2679,7 @@ term_to_binary(_Term) ->
 Term :: term(),
 Options :: [compressed |
 {compressed, Level :: 0..9} |
+ deterministic |
 {minor_version, Version :: 0..2} ].
 term_to_binary(_Term, _Options) ->
 erlang:nif_error(undefined).
@@ -2692,6 +2693,7 @@ term_to_iovec(_Term) ->
 Term :: term(),
 Options :: [compressed |
 {compressed, Level :: 0..9} |
+ deterministic |
 {minor_version, Version :: 0..2} ].
 term_to_iovec(_Term, _Options) ->
 erlang:nif_error(undefined).
diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl
index 8619056cbf..b32c15a270 100644
--- a/lib/compiler/src/beam_dict.erl
+++ b/lib/compiler/src/beam_dict.erl
@@ -299,7 +299,7 @@ my_term_to_binary(Term) ->
 %% options for the compiler. (When this comment was written, some time
 %% after the release of OTP 22, the default minor version was 1.)
 
- term_to_binary(Term, [{minor_version,2}]).
+ term_to_binary(Term, [{minor_version,2},deterministic]).
 
 %% Return the line table.
 -spec line_table(bdict()) ->
diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl
index 567e7e8f42..22b776c7f7 100644
--- a/lib/compiler/src/erl_bifs.erl
+++ b/lib/compiler/src/erl_bifs.erl
@@ -139,7 +139,6 @@ is_pure(erlang, round, 1) -> true;
 is_pure(erlang, setelement, 3) -> true;
 is_pure(erlang, size, 1) -> true;
 is_pure(erlang, split_binary, 2) -> true;
-is_pure(erlang, term_to_binary, 1) -> true;
 is_pure(erlang, tl, 1) -> true;
 is_pure(erlang, trunc, 1) -> true;
 is_pure(erlang, tuple_size, 1) -> true;
-- 
2.31.1

Places

File 0277-Ensure-reproducible-builds-of-Erlang-code.patch of Package erlang (Revision ab414368c0f53cdf471f751b019c6c4f)

Places