Merge 71787ed086 into c4703a7f19

builtin/common/metatable.lua

@@ -1,6 +1,6 @@
 -- Registered metatables, used by the C++ packer
 local known_metatables = {}
-function core.register_async_metatable(name, mt)
+function core.register_metatable(name, mt)
 	assert(type(name) == "string", ("attempt to use %s value as metatable name"):format(type(name)))
 	assert(type(mt) == "table", ("attempt to register a %s value as metatable"):format(type(mt)))
 	assert(known_metatables[name] == nil or known_metatables[name] == mt,
@@ -10,4 +10,4 @@ function core.register_async_metatable(name, mt)
 end
 core.known_metatables = known_metatables
 
-core.register_async_metatable("__builtin:vector", vector.metatable)
+core.register_metatable("__builtin:vector", vector.metatable)

builtin/common/serialize.lua

@@ -8,12 +8,23 @@ local next, rawget, pairs, pcall, error, type, setfenv, loadstring
 local table_concat, string_dump, string_format, string_match, math_huge
 	= table.concat, string.dump, string.format, string.match, math.huge
 
--- Recursively counts occurrences of objects (non-primitives including strings) in a table.
-local function count_objects(value)
+local itemstack_mt
+if ItemStack then
+	itemstack_mt = getmetatable(ItemStack())
+end
+local function is_itemstack(x)
+	return itemstack_mt and getmetatable(x) == itemstack_mt
+end
+
+-- Recursively
+-- (1) reads metatables from tables;
+-- (2) counts occurrences of objects (non-primitives including strings) in a table.
+local function prepare_objects(value)
 	local counts = {}
+	local type_lookup = {}
 	if value == nil then
 		-- Early return for nil; tables can't contain nil
-		return counts
+		return counts, type_lookup
 	end
 	local function count_values(val)
 		local type_ = type(val)
@@ -22,19 +33,23 @@ local function count_objects(value)
 		end
 		local count = counts[val]
 		counts[val] = (count or 0) + 1
+		local mt = getmetatable(val)
 		if type_ == "table" then
 			if not count then
 				for k, v in pairs(val) do
 					count_values(k)
 					count_values(v)
 				end
+				if mt then
+					type_lookup[val] = core.known_metatables[mt]
+				end
 			end
-		elseif type_ ~= "string" and type_ ~= "function" then
+		elseif type_ ~= "string" and type_ ~= "function" and not is_itemstack(val) then
 			error("unsupported type: " .. type_)
 		end
 	end
-	count_values(value)
-	return counts
+	count_values(value, {})
+	return counts, type_lookup
 end
 
 -- Build a "set" of Lua keywords. These can't be used as short key names.
@@ -58,6 +73,10 @@ local function dump_func(func)
 	return string_format("loadstring(%q)", string_dump(func))
 end
 
+local function dump_itemstack(item)
+	return string_format("ItemStack(%q)", item:to_string())
+end
+
 -- Serializes Lua nil, booleans, numbers, strings, tables and even functions
 -- Tables are referenced by reference, strings are referenced by value. Supports circular tables.
 local function serialize(value, write)
@@ -66,7 +85,11 @@ local function serialize(value, write)
 	local references = {}
 	-- Circular tables that must be filled using `table[key] = value` statements
 	local to_fill = {}
-	for object, count in pairs(count_objects(value)) do
+	local counts, typenames = prepare_objects(value)
+	if next(typenames) then
+		write "if not setmetatable then core={known_metatables={}}; setmetatable = function(x) return x end; end;"
+	end
+	for object, count in pairs(counts) do
 		local type_ = type(object)
 		-- Object must appear more than once. If it is a string, the reference has to be shorter than the string.
 		if count >= 2 and (type_ ~= "string" or #reference + 5 < #object) then
@@ -82,10 +105,12 @@ local function serialize(value, write)
 				write(dump_func(object))
 			elseif type_ == "string" then
 				write(quote(object))
+			elseif is_itemstack(object) then
+				write(dump_itemstack(object))
 			end
 			write(";")
 			references[object] = reference
-			if type_ == "table" then
+			if type_ ~= "string" and not is_itemstack(object) then
 				to_fill[object] = reference
 			end
 			refnum = refnum + 1
@@ -96,7 +121,7 @@ local function serialize(value, write)
 	local function use_short_key(key)
 		return not references[key] and type(key) == "string" and (not keywords[key]) and string_match(key, "^[%a_][%a%d_]*$")
 	end
-	local function dump(value)
+	local function dump(value, skip_mt)
 		-- Primitive types
 		if value == nil then
 			return write("nil")
@@ -126,12 +151,23 @@ local function serialize(value, write)
 			write(ref)
 			return write"]"
 		end
+		if (not skip_mt) and typenames[value] then
+			write "setmetatable("
+			dump(value, true)
+			write ",core.known_metatables["
+			dump(typenames[value])
+			write "] or {})"
+			return
+		end
 		if type_ == "string" then
 			return write(quote(value))
 		end
 		if type_ == "function" then
 			return write(dump_func(value))
 		end
+		if is_itemstack(value) then
+			return write(dump_itemstack(value))
+		end
 		if type_ == "table" then
 			write("{")
 			-- First write list keys:
@@ -169,6 +205,7 @@ local function serialize(value, write)
 	end
 	-- Write the statements to fill circular tables
 	for table, ref in pairs(to_fill) do
+		local typename = typenames[table]
 		for k, v in pairs(table) do
 			write("_[")
 			write(ref)
@@ -185,6 +222,13 @@ local function serialize(value, write)
 			dump(v)
 			write(";")
 		end
+		if typename then
+			write("setmetatable(_[")
+			write(ref)
+			write("],core.known_metatables[")
+			dump(typename)
+			write("] or {})")
+		end
 	end
 	write("return ")
 	dump(value)
@@ -216,7 +260,13 @@ function core.deserialize(str, safe)
 	if not func then return nil, err end
 
 	-- math.huge was serialized to inf and NaNs to nan by Lua in Minetest 5.6, so we have to support this here
-	local env = {inf = math_huge, nan = 0/0}
+	local env = {
+		inf = math_huge,
+		nan = 0/0,
+		ItemStack = ItemStack or function(str) return str end,
+		setmetatable = setmetatable,
+		core = { known_metatables = core.known_metatables }
+	}
 	if safe then
 		env.loadstring = dummy_func
 	else
@@ -236,3 +286,4 @@ function core.deserialize(str, safe)
 	end
 	return nil, value_or_err
 end
+

builtin/common/tests/serialize_spec.lua

@@ -4,6 +4,7 @@ _G.setfenv = require 'busted.compatibility'.setfenv
 
 dofile("builtin/common/serialize.lua")
 dofile("builtin/common/vector.lua")
+dofile("builtin/common/metatable.lua")
 
 -- Supports circular tables; does not support table keys
 -- Correctly checks whether a mapping of references ("same") exists
@@ -40,11 +41,32 @@ local t1, t2 = {x, x, y, y}, {x, y, x, y}
 assert.same(t1, t2) -- will succeed because it only checks whether the depths match
 assert(not pcall(assert_same, t1, t2)) -- will correctly fail because it checks whether the refs match
 
+local pair_mt = {
+	__eq = function(x, y)
+		return x[1] == y[1] and x[2] == y[2]
+	end,
+}
+local function pair(x, y)
+	return setmetatable({x, y}, pair_mt)
+end
+-- Use our own serialization functions to avoid incorrectly passing test related to references.
+core.register_metatable("pair", pair_mt)
+assert.equals(pair(1, 2), pair(1, 2))
+assert.not_equals(pair(1, 2), pair(3, 4))
+
 describe("serialize", function()
 	local function assert_preserves(value)
 		local preserved_value = core.deserialize(core.serialize(value))
 		assert_same(value, preserved_value)
 	end
+	local function assert_strictly_preserves(value)
+		local preserved_value = core.deserialize(core.serialize(value))
+		assert.equals(value, preserved_value)
+	end
+	local function assert_compatibly_preserves(value)
+		local preserved_value = loadstring(core.serialize(value))()
+		assert_same(value, preserved_value)
+	end
 	it("works", function()
 		assert_preserves({cat={sound="nyan", speed=400}, dog={sound="woof"}})
 	end)
@@ -53,6 +75,10 @@ describe("serialize", function()
 		assert_preserves({escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"})
 	end)
 
+	it("handles nil", function()
+		assert_strictly_preserves(nil)
+	end)
+
 	it("handles NaN & infinities", function()
 		local nan = core.deserialize(core.serialize(0/0))
 		assert(nan ~= nan)
@@ -113,7 +139,10 @@ describe("serialize", function()
 	it("vectors work", function()
 		local v = vector.new(1, 2, 3)
 		assert_preserves({v})
-		assert_preserves(v)
+		assert_compatibly_preserves({v})
+		assert_strictly_preserves(v)
+		assert_compatibly_preserves(v)
+		assert(core.deserialize(core.serialize(v)):check())
 
 		-- abuse
 		v = vector.new(1, 2, 3)
@@ -121,6 +150,43 @@ describe("serialize", function()
 		assert_preserves(v)
 	end)
 
+	it("correctly handles typed objects with multiple references", function()
+		local x, y = pair(1, 2), pair(1, 2)
+		local t = core.deserialize(core.serialize{x, x, y})
+		assert.equals(x, t[1])
+		assert.equals(x, t[3])
+		assert(rawequal(t[1], t[2]))
+		assert(not rawequal(t[1], t[3]))
+	end)
+
+	it("correctly handles recursive typed objects with the identity function as serializer", function()
+		local mt = {
+			__eq = function(x, y)
+				return x[1] == y[1]
+			end,
+		}
+		core.register_metatable("test_recursive_typed", mt)
+		local t = setmetatable({1}, mt)
+		t[2] = t
+		assert_strictly_preserves(t)
+	end)
+
+	it("correctly handles binary trees", function()
+		local child = {pair(1, 1)}
+		local layers = 4
+		for i = 2, layers do
+			child[i] = pair(child[i-1], child[i-1])
+		end
+		local tree = child[layers]
+		assert_strictly_preserves(tree)
+		local node = core.deserialize(core.serialize(tree))
+		for i = 2, layers do
+			assert(rawequal(node[1], node[2]))
+			node = node[1]
+		end
+		assert_compatibly_preserves(tree)
+	end)
+
 	it("handles keywords as keys", function()
 		assert_preserves({["and"] = "keyword", ["for"] = "keyword"})
 	end)

builtin/common/vector.lua

@@ -11,6 +11,10 @@ vector = {}
 local metatable = {}
 vector.metatable = metatable
 
+if core and core.register_serializable then
+	core.register_serializable("__builtin:vector", metatable)
+end
+
 local xyz = {"x", "y", "z"}
 
 -- only called when rawget(v, key) returns nil

doc/lua_api.md

@@ -6681,6 +6681,10 @@ Arguments and return values passed through this can contain certain userdata
 objects that will be seamlessly copied (not shared) to the async environment.
 This allows you easy interoperability for delegating work to jobs.
 
+Metatables are not kept across environments by default. Use `core.register_metatable`
+if you want a metatable to be kept. Note that you need to register the metatable
+in the main environment and in the async environment.
+
 * `minetest.handle_async(func, callback, ...)`:
     * Queue the function `func` to be ran in an async environment.
       Note that there are multiple persistent workers and any of them may
@@ -6693,17 +6697,6 @@ This allows you easy interoperability for delegating work to jobs.
     * Register a path to a Lua file to be imported when an async environment
       is initialized. You can use this to preload code which you can then call
       later using `minetest.handle_async()`.
-* `minetest.register_async_metatable(name, mt)`:
-    * Register a metatable that should be preserved when data is transferred
-    between the main thread and the async environment.
-    * `name` is a string that identifies the metatable. It is recommended to
-      follow the `modname:name` convention for this identifier.
-    * `mt` is the metatable to register.
-    * Note that it is allowed to register the same metatable under multiple
-      names, but it is not allowed to register multiple metatables under the
-      same name.
-    * You must register the metatable in both the main environment
-      and the async environment for this mechanism to work.
 
 
 ### List of APIs available in an async environment
@@ -6730,7 +6723,7 @@ Class instances that can be transferred between environments:
 Functions:
 * Standalone helpers such as logging, filesystem, encoding,
   hashing or compression APIs
-* `minetest.register_async_metatable` (see above)
+* `minetest.register_metatable` (see above)
 
 Variables:
 * `minetest.settings`
@@ -7134,6 +7127,19 @@ Misc.
     * Example: `deserialize('print("foo")')`, returns `nil`
       (function call fails), returns
       `error:[string "print("foo")"]:1: attempt to call global 'print' (a nil value)`
+* `minetest.register_metatable(name, mt)`:
+    * Register a metatable that should be preserved when data is transferred
+    across Lua environments, such as between the main and async environments
+    or for `minetest.serialize`.
+    * `name` is a string that identifies the metatable. It is recommended to
+      follow the `modname:name` convention for this identifier.
+    * `mt` is the metatable to register.
+    * Note that it is allowed to register the same metatable under multiple
+      names, but it is not allowed to register multiple metatables under the
+      same name.
+    * If you intend to use this for the async environment, you must register the
+      metatable in both the main environment and the async environment for this
+      mechanism to work.
 * `minetest.compress(data, method, ...)`: returns `compressed_data`
     * Compress a string of data.
     * `method` is a string identifying the compression method to be used.

games/devtest/mods/unittests/async_env.lua

@@ -168,17 +168,17 @@ end
 unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true})
 
 local function test_async_metatable_override()
-	assert(pcall(core.register_async_metatable, "__builtin:vector", vector.metatable),
+	assert(pcall(core.register_metatable, "__builtin:vector", vector.metatable),
 			"Metatable name aliasing throws an error when it should be allowed")
 
-	assert(not pcall(core.register_async_metatable, "__builtin:vector", {}),
+	assert(not pcall(core.register_metatable, "__builtin:vector", {}),
 			"Illegal metatable overriding allowed")
 end
 unittests.register("test_async_metatable_override", test_async_metatable_override)
 
 local function test_async_metatable_registration(cb)
 	local custom_metatable = {}
-	core.register_async_metatable("unittests:custom_metatable", custom_metatable)
+	core.register_metatable("unittests:custom_metatable", custom_metatable)
 
 	core.handle_async(function(x)
 		-- unittests.custom_metatable is registered in inside_async_env.lua

games/devtest/mods/unittests/inside_async_env.lua

@@ -3,7 +3,7 @@ unittests = {}
 core.log("info", "Hello World")
 
 unittests.custom_metatable = {}
-core.register_async_metatable("unittests:custom_metatable", unittests.custom_metatable)
+core.register_metatable("unittests:custom_metatable", unittests.custom_metatable)
 
 local function do_tests()
 	assert(core == minetest)

games/devtest/mods/unittests/itemstack_equals.lua

@@ -72,3 +72,10 @@ local function test_itemstack_equals_metadata()
 end
 
 unittests.register("test_itemstack_equals_metadata", test_itemstack_equals_metadata)
+
+local function test_itemstack_serialization_preservation()
+	local i = ItemStack("basenodes:stone 20 1000")
+	assert(i:equals(core.deserialize(core.serialize(i))))
+end
+
+unittests.register("test_itemstack_serialization_preservation", test_itemstack_serialization_preservation)