Compressing with lz4

Cargo.lock

@@ -15,6 +15,7 @@ dependencies = [
  "glob",
  "image",
  "linked-hash-map",
+ "lzzzz",
  "noise",
  "rand 0.8.3",
  "shaderc",
@@ -1154,6 +1155,15 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "lzzzz"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d891cedd3b1659c206a60ff8afd15bccd7c2754b157f8a164861989e042b88"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"

Cargo.toml

@@ -17,6 +17,7 @@ rand = "0.8"
 noise = "0.7"
 dirs = "3.0"
 flate2 = "1.0"
+lzzzz = "0.8"
 yaml-rust = "0.4"
 linked-hash-map = "0.5"
 

src/game/mod.rs

@@ -27,10 +27,19 @@ impl Game {
             world: World::new("default".to_string())
         };
 
-
+        // Try to load world from file
         let timer = Instant::now();
-        for x in 0..64 {
+        if app.world.load() {
-            for z in 0..64 {
+            for (pos, block) in app.world.all_blocks() {
+                content.write(&pos, block);
+            }
+            println!("Loaded in {}", timer.elapsed().as_secs_f32());
+        }
+        // Generate new world
+        else {
+            let timer = Instant::now();
+            for x in 0..32 {
+                for z in 0..32 {
                     let pos = &Point3{x,y:0,z};
                     let block = app.world.gen_block(pos);
                     content.write(pos, block);
@@ -41,13 +50,8 @@ impl Game {
             let timer = Instant::now();
             app.world.save();
             println!("Saved in {}", timer.elapsed().as_secs_f32());
+        }
         
-        // let timer = Instant::now();
-        // app.world.load();
-        // for (pos, block) in app.world.all_blocks() {
-        //     content.write(&pos, block);
-        // }
-        // println!("Loaded in {}", timer.elapsed().as_secs_f32());
 
         println!("App initialized");
         app

src/game/world/chunk.rs

@@ -7,6 +7,7 @@ use byteorder::ByteOrder;
 use cgmath::Point3;
 use std::io::Write;
 use std::convert::TryInto;
+use lzzzz::lz4;
 use std::io::Read;
 use crate::game::world::{WorldBlock, block};
 
@@ -29,21 +30,24 @@ impl WorldChunk {
 
     pub fn from_bytes(bytes: Vec<u8>) -> Self {
         let mut instance = Self::new();
-        let (node_bytes, block_bytes) = bytes.split_at((SIZE_QB*4) as usize);
+        let (header_bytes, bytes) = bytes.split_at(128);
+        let (node_bytes, block_bytes_compressed) = bytes.split_at((SIZE_QB*4) as usize);
+
+        // Header
+        let file_ver = header_bytes[0];
+        let num_blocks = LittleEndian::read_u32(&header_bytes[1..5]) as usize;
+        assert_eq!(file_ver, 1);
 
         // Nodes
-        let mut num_blocks = 0;
         for i in 0..SIZE_QB {
             let idx = (i*4) as usize;
             let val = u32::from_le_bytes([node_bytes[idx], node_bytes[idx+1], node_bytes[idx+2], node_bytes[idx+3]]);
             instance.nodes[i as usize] = val;
-            if val > 0 { num_blocks+=1; }
         }
 
         // Blocks
-        let mut decoder = DeflateDecoder::new(block_bytes);
+        let mut block_bytes = vec![0_u8; num_blocks*SIZE_QB];
-        let mut block_bytes = Vec::with_capacity(SIZE_QB);
+        lz4::decompress(&block_bytes_compressed, &mut block_bytes[0..num_blocks*SIZE_QB]).unwrap();
-        decoder.read_to_end(&mut block_bytes).expect("Failed to decode");
 
         for i in 0..num_blocks {
             let mut block = WorldBlock::new();
@@ -55,6 +59,35 @@ impl WorldChunk {
         instance
     }
 
+    pub fn to_bytes(&self) -> Vec<u8> {
+        
+        // Header
+        let mut bytes: Vec<u8> = vec![0_u8; 5];
+        bytes[0] = 1_u8; // 1b -> File version
+        LittleEndian::write_u32(&mut bytes[1..5], self.blocks.len() as u32); // 4b -> Number of blocks in this file
+        bytes.extend((0..128-5).map(|_| 0)); // Reserved 128b - 5b used
+
+        // Nodes
+        let mut node_bytes = [0_u8; (SIZE_QB*4) as usize];
+        LittleEndian::write_u32_into(&self.nodes, &mut node_bytes);
+        bytes.extend(node_bytes.iter());
+
+        // Blocks
+        let mut buf = Vec::new();
+        for block in self.blocks.iter() {
+            buf.extend(&block.materials);
+        }
+        lz4::compress_to_vec(&buf, &mut bytes, lz4::ACC_LEVEL_DEFAULT).unwrap();
+
+
+        // alg            |   size   |  compr  |  decom  
+        // raw              32.89kB     0.06s     1.19s
+        // deflate fast      1.23kB     1.22s     5.46s
+        // lz4               1.6kB      0.07s     1.23s
+
+        bytes
+    }
+
     pub fn all_blocks(&self) -> Vec<(Point3<u32>, &WorldBlock)> {
         self.nodes.iter().enumerate()
             .filter(|(_, n)| **n != 0)
@@ -87,22 +120,4 @@ impl WorldChunk {
         // Return block reference
         &mut self.blocks[(value-1) as usize]
     }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::with_capacity(self.nodes.len()*4 + self.blocks.len()* 32*32*32);
-        
-        // Nodes
-        let mut node_bytes = [0_u8; (SIZE_QB*4) as usize];
-        LittleEndian::write_u32_into(&self.nodes, &mut node_bytes);
-        bytes.extend(node_bytes.iter());
-
-        // Blocks
-        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::fast());
-        for block in self.blocks.iter() {
-            encoder.write_all(&block.materials).unwrap();
-        }
-        bytes.extend(encoder.finish().unwrap());
-    
-        bytes
-    }
 }

src/game/world/mod.rs

@@ -72,16 +72,17 @@ impl World {
         data_file.write_all(data_str.as_bytes()).expect("Failed to write");
     }
 
-    pub fn load(&mut self) {
+    pub fn load(&mut self) -> bool {
         // Get directory
         let mut dir = dirs::data_local_dir().unwrap();
         dir.push("Voxelgame");
         if !dir.is_dir() {
             println!("Not exists: {:?}", dir);
-            return; 
+            return false; 
         }
 
         // Load all chunks
+        let mut count = 0;
         for entry in fs::read_dir(dir).unwrap() {
             let path: PathBuf = entry.unwrap().path();
             if path.extension().unwrap().to_str() != Some("dat") {
@@ -102,10 +103,11 @@ impl World {
 
             println!("loaded {:?}", pos);
             self.chunks.insert(pos, chunk);
+            count += 1;
         }
 
         // Load metadata
-
+        return count > 0;
     }
 
     pub fn gen_block(&mut self, block_wpos: &Point3<u32>) -> &WorldBlock {