piotrek/first-voxels
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compressing with lz4

Browse Source
This commit is contained in:
Piotrek
2021-05-10 13:30:15 +02:00
parent 3bd9fa942e
commit 4eaa0fab56
5 changed files with 76 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+10
View File
@@ -15,6 +15,7 @@ dependencies = [
"glob",
"image",
"linked-hash-map",
"lzzzz",
"noise",
"rand 0.8.3",
"shaderc",
@@ -1154,6 +1155,15 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "lzzzz"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d891cedd3b1659c206a60ff8afd15bccd7c2754b157f8a164861989e042b88"
dependencies = [
"cc",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
Cargo.toml
+1
View File
@@ -17,6 +17,7 @@ rand = "0.8"
noise = "0.7"
dirs = "3.0"
flate2 = "1.0"
lzzzz = "0.8"
yaml-rust = "0.4"
linked-hash-map = "0.5"
src/game/mod.rs
+20 -16
View File
@@ -27,27 +27,31 @@ impl Game {
world: World::new("default".to_string())
};
// Try to load world from file
let timer = Instant::now();
for x in 0..64 {
for z in 0..64 {
let pos = &Point3{x,y:0,z};
let block = app.world.gen_block(pos);
content.write(pos, block);
if app.world.load() {
for (pos, block) in app.world.all_blocks() {
content.write(&pos, block);
}
println!("Loaded in {}", timer.elapsed().as_secs_f32());
}
println!("Generated in {}", timer.elapsed().as_secs_f32());
// Generate new world
else {
let timer = Instant::now();
for x in 0..32 {
for z in 0..32 {
let pos = &Point3{x,y:0,z};
let block = app.world.gen_block(pos);
content.write(pos, block);
}
}
println!("Generated in {}", timer.elapsed().as_secs_f32());
let timer = Instant::now();
app.world.save();
println!("Saved in {}", timer.elapsed().as_secs_f32());
let timer = Instant::now();
app.world.save();
println!("Saved in {}", timer.elapsed().as_secs_f32());
}
// let timer = Instant::now();
// app.world.load();
// for (pos, block) in app.world.all_blocks() {
// content.write(&pos, block);
// }
// println!("Loaded in {}", timer.elapsed().as_secs_f32());
println!("App initialized");
app
src/game/world/chunk.rs
+39 -24
View File
@@ -7,6 +7,7 @@ use byteorder::ByteOrder;
use cgmath::Point3;
use std::io::Write;
use std::convert::TryInto;
use lzzzz::lz4;
use std::io::Read;
use crate::game::world::{WorldBlock, block};
@@ -29,21 +30,24 @@ impl WorldChunk {
pub fn from_bytes(bytes: Vec<u8>) -> Self {
let mut instance = Self::new();
let (node_bytes, block_bytes) = bytes.split_at((SIZE_QB*4) as usize);
let (header_bytes, bytes) = bytes.split_at(128);
let (node_bytes, block_bytes_compressed) = bytes.split_at((SIZE_QB*4) as usize);
// Header
let file_ver = header_bytes[0];
let num_blocks = LittleEndian::read_u32(&header_bytes[1..5]) as usize;
assert_eq!(file_ver, 1);
// Nodes
let mut num_blocks = 0;
for i in 0..SIZE_QB {
let idx = (i*4) as usize;
let val = u32::from_le_bytes([node_bytes[idx], node_bytes[idx+1], node_bytes[idx+2], node_bytes[idx+3]]);
instance.nodes[i as usize] = val;
if val > 0 { num_blocks+=1; }
}
// Blocks
let mut decoder = DeflateDecoder::new(block_bytes);
let mut block_bytes = Vec::with_capacity(SIZE_QB);
decoder.read_to_end(&mut block_bytes).expect("Failed to decode");
let mut block_bytes = vec![0_u8; num_blocks*SIZE_QB];
lz4::decompress(&block_bytes_compressed, &mut block_bytes[0..num_blocks*SIZE_QB]).unwrap();
for i in 0..num_blocks {
let mut block = WorldBlock::new();
@@ -55,6 +59,35 @@ impl WorldChunk {
instance
}
pub fn to_bytes(&self) -> Vec<u8> {
// Header
let mut bytes: Vec<u8> = vec![0_u8; 5];
bytes[0] = 1_u8; // 1b -> File version
LittleEndian::write_u32(&mut bytes[1..5], self.blocks.len() as u32); // 4b -> Number of blocks in this file
bytes.extend((0..128-5).map(|_| 0)); // Reserved 128b - 5b used
// Nodes
let mut node_bytes = [0_u8; (SIZE_QB*4) as usize];
LittleEndian::write_u32_into(&self.nodes, &mut node_bytes);
bytes.extend(node_bytes.iter());
// Blocks
let mut buf = Vec::new();
for block in self.blocks.iter() {
buf.extend(&block.materials);
}
lz4::compress_to_vec(&buf, &mut bytes, lz4::ACC_LEVEL_DEFAULT).unwrap();
// alg | size | compr | decom
// raw 32.89kB 0.06s 1.19s
// deflate fast 1.23kB 1.22s 5.46s
// lz4 1.6kB 0.07s 1.23s
bytes
}
pub fn all_blocks(&self) -> Vec<(Point3<u32>, &WorldBlock)> {
self.nodes.iter().enumerate()
.filter(|(_, n)| **n != 0)
@@ -87,22 +120,4 @@ impl WorldChunk {
// Return block reference
&mut self.blocks[(value-1) as usize]
}
pub fn to_bytes(&self) -> Vec<u8> {
let mut bytes = Vec::with_capacity(self.nodes.len()*4 + self.blocks.len()* 32*32*32);
// Nodes
let mut node_bytes = [0_u8; (SIZE_QB*4) as usize];
LittleEndian::write_u32_into(&self.nodes, &mut node_bytes);
bytes.extend(node_bytes.iter());
// Blocks
let mut encoder = DeflateEncoder::new(Vec::new(), Compression::fast());
for block in self.blocks.iter() {
encoder.write_all(&block.materials).unwrap();
}
bytes.extend(encoder.finish().unwrap());
bytes
}
}
src/game/world/mod.rs
+5 -3
View File
@@ -72,16 +72,17 @@ impl World {
data_file.write_all(data_str.as_bytes()).expect("Failed to write");
}
pub fn load(&mut self) {
pub fn load(&mut self) -> bool {
// Get directory
let mut dir = dirs::data_local_dir().unwrap();
dir.push("Voxelgame");
if !dir.is_dir() {
println!("Not exists: {:?}", dir);
return;
return false;
}
// Load all chunks
let mut count = 0;
for entry in fs::read_dir(dir).unwrap() {
let path: PathBuf = entry.unwrap().path();
if path.extension().unwrap().to_str() != Some("dat") {
@@ -102,10 +103,11 @@ impl World {
println!("loaded {:?}", pos);
self.chunks.insert(pos, chunk);
count += 1;
}
// Load metadata
return count > 0;
}
pub fn gen_block(&mut self, block_wpos: &Point3<u32>) -> &WorldBlock {