Compress data files (reduces size by a factor 3-4)

init.lua

@@ -24,22 +24,22 @@ local X = tonumber(sfile:read('*l'))
 local Z = tonumber(sfile:read('*l'))
 
 copy_if_needed('dem')
-local dem = load_map(worldpath..'dem', 2, true)
+local dem = load_map(worldpath..'dem', 2, true, X*Z)
 copy_if_needed('lakes')
-local lakes = load_map(worldpath..'lakes', 2, true)
+local lakes = load_map(worldpath..'lakes', 2, true, X*Z)
 copy_if_needed('bounds_x')
-local bounds_x = load_map(worldpath..'bounds_x', 4, false)
+local bounds_x = load_map(worldpath..'bounds_x', 4, false, (X-1)*Z)
 copy_if_needed('bounds_y')
-local bounds_z = load_map(worldpath..'bounds_y', 4, false)
+local bounds_z = load_map(worldpath..'bounds_y', 4, false, X*(Z-1))
 
 copy_if_needed('offset_x')
-local offset_x = load_map(worldpath..'offset_x', 1, true)
+local offset_x = load_map(worldpath..'offset_x', 1, true, X*Z)
 for k, v in ipairs(offset_x) do
 	offset_x[k] = (v+0.5)/256
 end
 
 copy_if_needed('offset_y')
-local offset_z = load_map(worldpath..'offset_y', 1, true)
+local offset_z = load_map(worldpath..'offset_y', 1, true, X*Z)
 for k, v in ipairs(offset_z) do
 	offset_z[k] = (v+0.5)/256
 end

load.lua

@@ -1,11 +1,12 @@
-local function load_map(filename, bytes, signed)
+local function load_map(filename, bytes, signed, size)
 	local file = io.open(filename, 'r')
 	local data = file:read('*all')
+	if #data < bytes*size then
+		data = minetest.decompress(data)
+	end
 
 	local map = {}
 
-	local size = math.floor(#data/bytes)
-
 	for i=1, size do
 		local i0, i1 = (i-1)*bytes+1, i*bytes
 		local elements = {data:byte(i0, i1)}

save.py

@@ -1,8 +1,13 @@
 import numpy as np
+import zlib
 
 def save(data, fname, dtype=None):
     if dtype is not None:
         data = data.astype(dtype)
 
+    bin_data = data.tobytes()
+    bin_data_comp = zlib.compress(bin_data, 9)
+    if len(bin_data_comp) < len(bin_data):
+        bin_data = bin_data_comp
     with open(fname, 'wb') as f:
-        f.write(data.tobytes())
+        f.write(bin_data)

view_map.py

@@ -1,13 +1,22 @@
 #!/usr/bin/env python3
 
 import numpy as np
+import zlib
 import matplotlib.pyplot as plt
 
+def load_map(name, dtype, shape):
+    dtype = np.dtype(dtype)
+    with open(name, 'rb') as f:
+        data = f.read()
+    if len(data) < shape[0]*shape[1]*dtype.itemsize:
+        data = zlib.decompress(data)
+    return np.frombuffer(data, dtype=dtype).reshape(shape)
+
 shape = np.loadtxt('size', dtype='u4')
 n = shape[0] * shape[1]
-dem = np.fromfile('dem', dtype='>i2').reshape(shape)
-lakes = np.fromfile('lakes', dtype='>i2').reshape(shape)
-rivers = np.fromfile('rivers', dtype='>u4').reshape(shape)
+dem = load_map('dem', '>i2', shape)
+lakes = load_map('lakes', '>i2', shape)
+rivers = load_map('rivers', '>u4', shape)
 
 plt.subplot(1,3,1)
 plt.pcolormesh(dem, cmap='viridis')