From 6e75fff4e4ac4a9dca039273ff7412f753904db3 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Thu, 28 Dec 2023 18:36:20 +0100 Subject: [PATCH] Implement GLTF material update in unpacker- first part --- Cargo.lock | 253 +++++++++++++++++++++++++++++++++++++----------- Cargo.toml | 2 +- src/asset.rs | 23 ++++- src/main.rs | 6 +- src/unpacker.rs | 56 ++++++++--- 5 files changed, 266 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0ddadc..1cb96c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,24 +8,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "anstream" version = "0.6.5" @@ -80,6 +62,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -92,6 +80,18 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cfg-if" version = "1.0.0" @@ -138,6 +138,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.0" @@ -201,6 +207,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fdeflate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209098dd6dfc4445aa6111f0e98653ac323eaa4dfd212c9ca3931bf9955c31bd" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filetime" version = "0.2.23" @@ -224,14 +239,42 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.14.3" +name = "gltf" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "3b78f069cf941075835822953c345b9e1edd67ae347b81ace3aea9de38c2ef33" dependencies = [ - "ahash", - "allocator-api2", - "rayon", + "base64", + "byteorder", + "gltf-json", + "image", + "lazy_static", + "serde_json", + "urlencoding", +] + +[[package]] +name = "gltf-derive" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "438ffe1a5540d75403feaf23636b164e816e93f6f03131674722b3886ce32a57" +dependencies = [ + "inflections", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "gltf-json" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655951ba557f2bc69ea4b0799446bae281fa78efae6319968bdd2c3e9a06d8e1" +dependencies = [ + "gltf-derive", + "serde", + "serde_derive", + "serde_json", ] [[package]] @@ -240,6 +283,45 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", +] + +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.151" @@ -254,11 +336,11 @@ checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lwa_unity_unpack" -version = "0.2.1" +version = "0.3.0" dependencies = [ "clap", "flate2", - "hashbrown", + "gltf", "rayon", "tar", ] @@ -279,13 +361,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", ] [[package]] -name = "once_cell" -version = "1.19.0" +name = "num-integer" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "png" +version = "0.17.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] [[package]] name = "proc-macro2" @@ -347,6 +467,49 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "strsim" version = "0.10.0" @@ -381,18 +544,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "windows-sys" version = "0.52.0" @@ -469,23 +632,3 @@ dependencies = [ "linux-raw-sys", "rustix", ] - -[[package]] -name = "zerocopy" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/Cargo.toml b/Cargo.toml index fa04568..11e4528 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,6 @@ opt-level = 2 [dependencies] clap = { version = "4.4", features = ["derive"] } flate2 = "1.0" -hashbrown = { version ="0.14.3", features = ["ahash","allocator-api2","inline-more","rayon"] } +gltf = "1.4.0" rayon = "1.8.0" tar = "0.4" diff --git a/src/asset.rs b/src/asset.rs index 5d10514..99451e3 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -4,7 +4,7 @@ use std::fs::DirEntry; use std::fs::File; use std::io::BufRead; use std::io::BufReader; -use std::path::{Path, PathBuf}; +use std::path::Path; #[derive(Clone)] pub struct Asset { @@ -12,6 +12,16 @@ pub struct Asset { pub hash: String, pub path_name: String, pub has_meta: bool, + pub asset_type: AssetType +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub enum AssetType{ + FbxModel, + Material, + Prefab, + Scene, + Other(String) } impl Asset { @@ -52,11 +62,22 @@ impl Asset { } } if has_asset { + let asset_type = match &extension { + Some(str) => match str.as_str() { + "fbx" => AssetType::FbxModel, + "prefab" => AssetType::Prefab, + "unity" => AssetType::Scene, + "mat" => AssetType::Material, + _ => AssetType::Other(str.clone()) + }, + _ => AssetType::Other(String::new()) + }; Some(Asset { extension, hash: asset, path_name: real_path, has_meta, + asset_type, }) } else { None diff --git a/src/main.rs b/src/main.rs index a962462..dcf3127 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,9 +6,13 @@ use clap::Parser; fn main() { let args = crate::args::Args::parse(); - let mut unpacker = crate::unpacker::Unpacker { args, assets: vec![] }; + let mut unpacker = crate::unpacker::Unpacker { + args, + assets: vec![], + }; unpacker.prepare_environment(); unpacker.extract(); unpacker.process_data(); + unpacker.update_gltf_materials(); } diff --git a/src/unpacker.rs b/src/unpacker.rs index e37cab4..12ff41a 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -1,21 +1,19 @@ -use crate::asset::Asset; +use crate::asset::{Asset,AssetType}; use flate2::read::GzDecoder; -use hashbrown::HashMap; use rayon::prelude::*; -use std::ffi::OsStr; use std::fs::File; -use std::io::BufRead; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::mpsc::channel; use std::sync::Arc; use std::{fs, io}; +use std::hash::Hash; use tar::Archive; #[derive(Clone)] pub struct Unpacker { pub args: crate::args::Args, - pub assets: Vec + pub assets: Vec, } impl Unpacker { @@ -62,6 +60,35 @@ impl Unpacker { self.assets = receiver.iter().collect(); } + pub fn update_gltf_materials(&self) { + if self.args.fbx_to_gltf.is_none() { + return; + } + let output_dir = Path::new(&self.args.output); + + let fbx_models : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::FbxModel).collect(); + let prefabs : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Prefab).collect(); + let materials : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Material).collect(); + println!("There are {} models, {} prefabs and {} materials", fbx_models.len(), prefabs.len(), materials.len()); + for prefab in prefabs.iter() { + let path = Path::new(&prefab.path_name); + let result_path = output_dir.join(path); + let prefab_content = fs::read_to_string(&result_path).unwrap(); + let matching_materials : Vec = materials.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); + let matching_models : Vec = fbx_models.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); + println!("Prefab: {},\nMaterials: ",&prefab.path_name); + for m in matching_materials.iter() { + println!(" - {}",&m.path_name); + } + println!("Models: "); + for m in matching_models.iter() { + println!(" - {}",&m.path_name); + } + + // now if there is one material and one model we should read material texture path and assign it to model material texture + } + } + pub fn process_data(&self) { let output_dir = Path::new(&self.args.output); let copy_meta_files = self.args.copy_meta_files; @@ -91,18 +118,15 @@ impl Unpacker { panic!("SOURCE ASSET DOES NOT EXIST: {}", source_asset.display()); } - if self.args.fbx_to_gltf.is_some() { - if let Some("fbx") = path.extension().and_then(OsStr::to_str) { - process_fbx_file( - &source_asset, - &result_path, - &self.args.fbx_to_gltf.clone().unwrap(), - ); - return; - } + if self.args.fbx_to_gltf.is_some() && &asset.asset_type == &AssetType::FbxModel { + process_fbx_file( + &source_asset, + &result_path, + &self.args.fbx_to_gltf.clone().unwrap(), + ); + } else { + process_non_fbx_file(&source_asset, &result_path); } - - process_non_fbx_file(&source_asset, &result_path); }); fs::remove_dir_all(Path::new(&*tmp_dir)).unwrap();