From 08aecd4004b343daf7f3db1ee5d7808d1c40f05c Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Thu, 28 Dec 2023 17:42:01 +0100 Subject: [PATCH 01/10] extract fn, store assets in Unpacker struct --- src/main.rs | 3 ++- src/unpacker.rs | 29 +++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index 785f87e..a962462 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,8 +6,9 @@ use clap::Parser; fn main() { let args = crate::args::Args::parse(); - let unpacker = crate::unpacker::Unpacker { args }; + let mut unpacker = crate::unpacker::Unpacker { args, assets: vec![] }; unpacker.prepare_environment(); + unpacker.extract(); unpacker.process_data(); } diff --git a/src/unpacker.rs b/src/unpacker.rs index d33eceb..e37cab4 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -5,7 +5,6 @@ use rayon::prelude::*; use std::ffi::OsStr; use std::fs::File; use std::io::BufRead; -use std::io::BufReader; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::mpsc::channel; @@ -16,6 +15,7 @@ use tar::Archive; #[derive(Clone)] pub struct Unpacker { pub args: crate::args::Args, + pub assets: Vec } impl Unpacker { @@ -35,12 +35,10 @@ impl Unpacker { fs::remove_dir_all(output_dir).unwrap(); } } - - pub fn process_data(&self) { + pub fn extract(&mut self) { let archive_path = Path::new(&self.args.input); - let output_dir = Path::new(&self.args.output); - let copy_meta_files = self.args.copy_meta_files; let tmp_path = Path::new("./tmp_dir"); + if let Err(e) = Unpacker::extract_archive(archive_path, tmp_path) { println!("Failed to extract archive: {}", e); } @@ -61,12 +59,18 @@ impl Unpacker { } } }); + self.assets = receiver.iter().collect(); + } + pub fn process_data(&self) { + let output_dir = Path::new(&self.args.output); + let copy_meta_files = self.args.copy_meta_files; + let tmp_path = Path::new("./tmp_dir"); + + let mapping_arc = Arc::new(&self.assets); let tmp_dir = Arc::new(tmp_path); fs::create_dir(output_dir).unwrap(); let output_dir = Arc::new(output_dir); - let mapping: Vec = receiver.iter().collect(); - let mapping_arc = Arc::new(mapping); mapping_arc.par_iter().for_each(|asset| { let asset_hash = &asset.hash; @@ -82,7 +86,10 @@ impl Unpacker { let result_path = output_dir.join(meta_path); fs::rename(source_meta, result_path).unwrap(); } - check_source_asset_exists(&source_asset); + + if !source_asset.exists() { + panic!("SOURCE ASSET DOES NOT EXIST: {}", source_asset.display()); + } if self.args.fbx_to_gltf.is_some() { if let Some("fbx") = path.extension().and_then(OsStr::to_str) { @@ -108,12 +115,6 @@ impl Unpacker { } } - fn check_source_asset_exists(source_asset: &Path) { - if !source_asset.exists() { - panic!("SOURCE ASSET DOES NOT EXIST: {}", source_asset.display()); - } - } - fn process_fbx_file(source_asset: &Path, result_path: &Path, tool: &PathBuf) { let out_path = result_path.with_extension(""); println!( From 6e75fff4e4ac4a9dca039273ff7412f753904db3 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Thu, 28 Dec 2023 18:36:20 +0100 Subject: [PATCH 02/10] Implement GLTF material update in unpacker- first part --- Cargo.lock | 253 +++++++++++++++++++++++++++++++++++++----------- Cargo.toml | 2 +- src/asset.rs | 23 ++++- src/main.rs | 6 +- src/unpacker.rs | 56 ++++++++--- 5 files changed, 266 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0ddadc..1cb96c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,24 +8,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "anstream" version = "0.6.5" @@ -80,6 +62,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -92,6 +80,18 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cfg-if" version = "1.0.0" @@ -138,6 +138,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.0" @@ -201,6 +207,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fdeflate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209098dd6dfc4445aa6111f0e98653ac323eaa4dfd212c9ca3931bf9955c31bd" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filetime" version = "0.2.23" @@ -224,14 +239,42 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.14.3" +name = "gltf" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "3b78f069cf941075835822953c345b9e1edd67ae347b81ace3aea9de38c2ef33" dependencies = [ - "ahash", - "allocator-api2", - "rayon", + "base64", + "byteorder", + "gltf-json", + "image", + "lazy_static", + "serde_json", + "urlencoding", +] + +[[package]] +name = "gltf-derive" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "438ffe1a5540d75403feaf23636b164e816e93f6f03131674722b3886ce32a57" +dependencies = [ + "inflections", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "gltf-json" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655951ba557f2bc69ea4b0799446bae281fa78efae6319968bdd2c3e9a06d8e1" +dependencies = [ + "gltf-derive", + "serde", + "serde_derive", + "serde_json", ] [[package]] @@ -240,6 +283,45 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", +] + +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.151" @@ -254,11 +336,11 @@ checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lwa_unity_unpack" -version = "0.2.1" +version = "0.3.0" dependencies = [ "clap", "flate2", - "hashbrown", + "gltf", "rayon", "tar", ] @@ -279,13 +361,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", ] [[package]] -name = "once_cell" -version = "1.19.0" +name = "num-integer" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "png" +version = "0.17.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] [[package]] name = "proc-macro2" @@ -347,6 +467,49 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "strsim" version = "0.10.0" @@ -381,18 +544,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "windows-sys" version = "0.52.0" @@ -469,23 +632,3 @@ dependencies = [ "linux-raw-sys", "rustix", ] - -[[package]] -name = "zerocopy" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/Cargo.toml b/Cargo.toml index fa04568..11e4528 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,6 @@ opt-level = 2 [dependencies] clap = { version = "4.4", features = ["derive"] } flate2 = "1.0" -hashbrown = { version ="0.14.3", features = ["ahash","allocator-api2","inline-more","rayon"] } +gltf = "1.4.0" rayon = "1.8.0" tar = "0.4" diff --git a/src/asset.rs b/src/asset.rs index 5d10514..99451e3 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -4,7 +4,7 @@ use std::fs::DirEntry; use std::fs::File; use std::io::BufRead; use std::io::BufReader; -use std::path::{Path, PathBuf}; +use std::path::Path; #[derive(Clone)] pub struct Asset { @@ -12,6 +12,16 @@ pub struct Asset { pub hash: String, pub path_name: String, pub has_meta: bool, + pub asset_type: AssetType +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub enum AssetType{ + FbxModel, + Material, + Prefab, + Scene, + Other(String) } impl Asset { @@ -52,11 +62,22 @@ impl Asset { } } if has_asset { + let asset_type = match &extension { + Some(str) => match str.as_str() { + "fbx" => AssetType::FbxModel, + "prefab" => AssetType::Prefab, + "unity" => AssetType::Scene, + "mat" => AssetType::Material, + _ => AssetType::Other(str.clone()) + }, + _ => AssetType::Other(String::new()) + }; Some(Asset { extension, hash: asset, path_name: real_path, has_meta, + asset_type, }) } else { None diff --git a/src/main.rs b/src/main.rs index a962462..dcf3127 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,9 +6,13 @@ use clap::Parser; fn main() { let args = crate::args::Args::parse(); - let mut unpacker = crate::unpacker::Unpacker { args, assets: vec![] }; + let mut unpacker = crate::unpacker::Unpacker { + args, + assets: vec![], + }; unpacker.prepare_environment(); unpacker.extract(); unpacker.process_data(); + unpacker.update_gltf_materials(); } diff --git a/src/unpacker.rs b/src/unpacker.rs index e37cab4..12ff41a 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -1,21 +1,19 @@ -use crate::asset::Asset; +use crate::asset::{Asset,AssetType}; use flate2::read::GzDecoder; -use hashbrown::HashMap; use rayon::prelude::*; -use std::ffi::OsStr; use std::fs::File; -use std::io::BufRead; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::mpsc::channel; use std::sync::Arc; use std::{fs, io}; +use std::hash::Hash; use tar::Archive; #[derive(Clone)] pub struct Unpacker { pub args: crate::args::Args, - pub assets: Vec + pub assets: Vec, } impl Unpacker { @@ -62,6 +60,35 @@ impl Unpacker { self.assets = receiver.iter().collect(); } + pub fn update_gltf_materials(&self) { + if self.args.fbx_to_gltf.is_none() { + return; + } + let output_dir = Path::new(&self.args.output); + + let fbx_models : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::FbxModel).collect(); + let prefabs : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Prefab).collect(); + let materials : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Material).collect(); + println!("There are {} models, {} prefabs and {} materials", fbx_models.len(), prefabs.len(), materials.len()); + for prefab in prefabs.iter() { + let path = Path::new(&prefab.path_name); + let result_path = output_dir.join(path); + let prefab_content = fs::read_to_string(&result_path).unwrap(); + let matching_materials : Vec = materials.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); + let matching_models : Vec = fbx_models.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); + println!("Prefab: {},\nMaterials: ",&prefab.path_name); + for m in matching_materials.iter() { + println!(" - {}",&m.path_name); + } + println!("Models: "); + for m in matching_models.iter() { + println!(" - {}",&m.path_name); + } + + // now if there is one material and one model we should read material texture path and assign it to model material texture + } + } + pub fn process_data(&self) { let output_dir = Path::new(&self.args.output); let copy_meta_files = self.args.copy_meta_files; @@ -91,18 +118,15 @@ impl Unpacker { panic!("SOURCE ASSET DOES NOT EXIST: {}", source_asset.display()); } - if self.args.fbx_to_gltf.is_some() { - if let Some("fbx") = path.extension().and_then(OsStr::to_str) { - process_fbx_file( - &source_asset, - &result_path, - &self.args.fbx_to_gltf.clone().unwrap(), - ); - return; - } + if self.args.fbx_to_gltf.is_some() && &asset.asset_type == &AssetType::FbxModel { + process_fbx_file( + &source_asset, + &result_path, + &self.args.fbx_to_gltf.clone().unwrap(), + ); + } else { + process_non_fbx_file(&source_asset, &result_path); } - - process_non_fbx_file(&source_asset, &result_path); }); fs::remove_dir_all(Path::new(&*tmp_dir)).unwrap(); From 8ef65376b8f5395f3987dfedc2e37e6091f49125 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Thu, 28 Dec 2023 20:41:49 +0100 Subject: [PATCH 03/10] Find texture path now missing part is reading gltf file, changing materials texture and saving it --- Cargo.toml | 1 + src/asset.rs | 70 ++++++++++++++++++++++++++------ src/unpacker.rs | 105 ++++++++++++++++++++++++++++++------------------ 3 files changed, 125 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 11e4528..16d68ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,4 +24,5 @@ clap = { version = "4.4", features = ["derive"] } flate2 = "1.0" gltf = "1.4.0" rayon = "1.8.0" +regex = "1.10.2" tar = "0.4" diff --git a/src/asset.rs b/src/asset.rs index 99451e3..29a8412 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -1,3 +1,4 @@ +use regex::Regex; use std::ffi::OsStr; use std::fs; use std::fs::DirEntry; @@ -5,32 +6,71 @@ use std::fs::File; use std::io::BufRead; use std::io::BufReader; use std::path::Path; +use std::path::PathBuf; #[derive(Clone)] pub struct Asset { pub extension: Option, - pub hash: String, - pub path_name: String, + pub guid: String, + pub path: String, pub has_meta: bool, - pub asset_type: AssetType + pub asset_type: AssetType, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -pub enum AssetType{ +pub enum AssetType { FbxModel, Material, Prefab, Scene, - Other(String) + Other(String), } impl Asset { - pub fn from_path(entry: &DirEntry) -> Option { + pub fn try_get_mat_texture_guid(&self) -> Option { + match &self.asset_type { + AssetType::Material => {} + _ => return None, + } + let file = File::open(&self.path).unwrap(); + let buf_reader = BufReader::new(file); + let search = buf_reader.lines().into_iter().find(|s| { + let ss = s.as_ref().unwrap(); + return ss.contains("m_Texture") && ss.contains("guid: "); + }); + if let Some(line) = search { + let line = line.unwrap_or_default(); + return extract_guid(&line); + } + None + } + + pub fn prepare_directory(&self) { + println!("{}: {:?}", self.guid, self.path); + let base_path = Path::new(&self.path); + let result_dir = base_path.parent(); + if result_dir.is_none() { + eprintln!("{} is none", &self.path); + } + let result_dir = result_dir.unwrap(); + if !result_dir.exists() { + let result = fs::create_dir_all(result_dir); + if result.is_err() { + eprintln!( + "Error {}: {}", + result_dir.to_str().unwrap(), + result.err().unwrap() + ); + } + } + } + + pub fn from_path(entry: &DirEntry, output_dir: &PathBuf) -> Option { let root_file = entry.path(); if !root_file.is_dir() { return None; } - let asset = entry.file_name().into_string().unwrap(); + let guid = entry.file_name().into_string().unwrap(); let mut real_path = String::new(); let mut extension = None; let mut has_asset = false; @@ -46,7 +86,7 @@ impl Asset { let line = buf_reader.lines().next(); match line { Some(Ok(path)) => { - real_path = path; + real_path = output_dir.join(path).to_str().unwrap().to_string(); if let Some(e) = Path::new(&real_path).extension().and_then(OsStr::to_str) { @@ -68,14 +108,14 @@ impl Asset { "prefab" => AssetType::Prefab, "unity" => AssetType::Scene, "mat" => AssetType::Material, - _ => AssetType::Other(str.clone()) + _ => AssetType::Other(str.clone()), }, - _ => AssetType::Other(String::new()) + _ => AssetType::Other(String::new()), }; Some(Asset { extension, - hash: asset, - path_name: real_path, + guid, + path: real_path, has_meta, asset_type, }) @@ -84,3 +124,9 @@ impl Asset { } } } + +fn extract_guid(text: &str) -> Option { + let re = Regex::new(r"guid: (?P[A-Za-z0-9]{32})").unwrap(); + re.captures(text) + .and_then(|cap| cap.name("guid").map(|guid| guid.as_str().to_string())) +} diff --git a/src/unpacker.rs b/src/unpacker.rs index 12ff41a..2129367 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -1,4 +1,4 @@ -use crate::asset::{Asset,AssetType}; +use crate::asset::{Asset, AssetType}; use flate2::read::GzDecoder; use rayon::prelude::*; use std::fs::File; @@ -7,7 +7,6 @@ use std::process::Command; use std::sync::mpsc::channel; use std::sync::Arc; use std::{fs, io}; -use std::hash::Hash; use tar::Archive; #[derive(Clone)] @@ -36,6 +35,7 @@ impl Unpacker { pub fn extract(&mut self) { let archive_path = Path::new(&self.args.input); let tmp_path = Path::new("./tmp_dir"); + let output_dir = Path::new(&self.args.output); if let Err(e) = Unpacker::extract_archive(archive_path, tmp_path) { println!("Failed to extract archive: {}", e); @@ -49,7 +49,7 @@ impl Unpacker { .par_bridge() .for_each_with(sender, |s, entry| { let entry = entry.unwrap(); - let asset = crate::asset::Asset::from_path(&entry); + let asset = crate::asset::Asset::from_path(&entry, &output_dir.to_path_buf()); if let Some(asset) = asset { let extension = &asset.extension.clone().unwrap_or_default(); if !ignored_extensions.contains(extension) { @@ -64,29 +64,67 @@ impl Unpacker { if self.args.fbx_to_gltf.is_none() { return; } - let output_dir = Path::new(&self.args.output); - - let fbx_models : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::FbxModel).collect(); - let prefabs : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Prefab).collect(); - let materials : Vec = self.assets.clone().into_iter().filter(|a| &a.asset_type == &AssetType::Material).collect(); - println!("There are {} models, {} prefabs and {} materials", fbx_models.len(), prefabs.len(), materials.len()); + let fbx_models: Vec = self + .assets + .clone() + .into_iter() + .filter(|a| &a.asset_type == &AssetType::FbxModel) + .collect(); + let prefabs: Vec = self + .assets + .clone() + .into_iter() + .filter(|a| &a.asset_type == &AssetType::Prefab) + .collect(); + let materials: Vec = self + .assets + .clone() + .into_iter() + .filter(|a| &a.asset_type == &AssetType::Material) + .collect(); + println!( + "There are {} models, {} prefabs and {} materials", + fbx_models.len(), + prefabs.len(), + materials.len() + ); + let mut counter = 0; + let mut result_models : Vec = vec![]; for prefab in prefabs.iter() { - let path = Path::new(&prefab.path_name); - let result_path = output_dir.join(path); - let prefab_content = fs::read_to_string(&result_path).unwrap(); - let matching_materials : Vec = materials.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); - let matching_models : Vec = fbx_models.clone().into_iter().filter(|a| prefab_content.contains(&a.hash)).collect(); - println!("Prefab: {},\nMaterials: ",&prefab.path_name); - for m in matching_materials.iter() { - println!(" - {}",&m.path_name); + let path = Path::new(&prefab.path); + let prefab_content = fs::read_to_string(&path).unwrap(); + let matching_materials: Vec = materials + .clone() + .into_iter() + .filter(|a| prefab_content.contains(&a.guid)) + .collect(); + let matching_models: Vec = fbx_models + .clone() + .into_iter() + .filter(|a| prefab_content.contains(&a.guid)) + .collect(); + if matching_materials.len() != 1 || 1 != matching_models.len() { + continue; } - println!("Models: "); - for m in matching_models.iter() { - println!(" - {}",&m.path_name); + let material = matching_materials.first().unwrap(); + let model = matching_models.first().unwrap(); + if result_models.iter().any(|a| model.guid.eq(&a.guid)) { + continue; } + let texture_guid: Option = material.try_get_mat_texture_guid(); + let texture_asset: &Asset; + match &texture_guid { + Some(guid) => { + texture_asset = self.assets.iter().find(|a| guid.eq(&a.guid)).unwrap() + } + None => continue, + } + // here we should read gltf file and replace material texture with Uri based on texture_asset - // now if there is one material and one model we should read material texture path and assign it to model material texture + result_models.push(model.clone()); + counter += 1; } + println!("Updated {} models", counter); } pub fn process_data(&self) { @@ -97,21 +135,18 @@ impl Unpacker { let mapping_arc = Arc::new(&self.assets); let tmp_dir = Arc::new(tmp_path); fs::create_dir(output_dir).unwrap(); - let output_dir = Arc::new(output_dir); mapping_arc.par_iter().for_each(|asset| { - let asset_hash = &asset.hash; - let path = Path::new(&asset.path_name); + let asset_hash = &asset.guid; + let path = Path::new(&asset.path); let source_asset = Path::new(&*tmp_dir).join(asset_hash).join("asset"); - let result_path = output_dir.join(path); - process_directory(asset_hash, &asset.path_name, &result_path); + asset.prepare_directory(); if copy_meta_files && asset.has_meta { let source_meta = Path::new(&*tmp_dir).join(asset_hash).join("asset.meta"); - let mut meta_path = asset.path_name.clone(); + let mut meta_path = asset.path.clone(); meta_path.push_str(".meta"); - let result_path = output_dir.join(meta_path); - fs::rename(source_meta, result_path).unwrap(); + fs::rename(source_meta, meta_path).unwrap(); } if !source_asset.exists() { @@ -121,24 +156,16 @@ impl Unpacker { if self.args.fbx_to_gltf.is_some() && &asset.asset_type == &AssetType::FbxModel { process_fbx_file( &source_asset, - &result_path, + &path, &self.args.fbx_to_gltf.clone().unwrap(), ); } else { - process_non_fbx_file(&source_asset, &result_path); + process_non_fbx_file(&source_asset, &path); } }); fs::remove_dir_all(Path::new(&*tmp_dir)).unwrap(); - fn process_directory(asset_hash: &str, asset_path: &str, result_path: &Path) { - println!("{}: {:?}", asset_hash, asset_path); - let result_dir = result_path.parent().unwrap(); - if !result_dir.exists() { - fs::create_dir_all(result_dir).unwrap(); - } - } - fn process_fbx_file(source_asset: &Path, result_path: &Path, tool: &PathBuf) { let out_path = result_path.with_extension(""); println!( From 4868671e7807c2afc3aa39c699f80815da41b6ae Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:00:48 +0100 Subject: [PATCH 04/10] Implement material modification in model unpacking --- Cargo.lock | 52 ++++++++++++++++++++++++++++++ src/unpacker.rs | 86 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cb96c9..96edb54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.5" @@ -341,10 +350,18 @@ dependencies = [ "clap", "flate2", "gltf", + "pathdiff", "rayon", + "regex", "tar", ] +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + [[package]] name = "memoffset" version = "0.9.0" @@ -394,6 +411,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "png" version = "0.17.10" @@ -454,6 +477,35 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "rustix" version = "0.38.28" diff --git a/src/unpacker.rs b/src/unpacker.rs index 2129367..fa9b25c 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -1,6 +1,9 @@ use crate::asset::{Asset, AssetType}; use flate2::read::GzDecoder; +use gltf::{json, Document}; use rayon::prelude::*; +use std::borrow::Cow; + use std::fs::File; use std::path::{Path, PathBuf}; use std::process::Command; @@ -15,6 +18,10 @@ pub struct Unpacker { pub assets: Vec, } +fn get_relative_path(path1: &Path, path2: &Path) -> Option { + pathdiff::diff_paths(path1, path2) +} + impl Unpacker { pub fn prepare_environment(&self) { let archive_path = Path::new(&self.args.input); @@ -88,11 +95,10 @@ impl Unpacker { prefabs.len(), materials.len() ); - let mut counter = 0; - let mut result_models : Vec = vec![]; - for prefab in prefabs.iter() { + + prefabs.par_iter().for_each(|prefab|{ let path = Path::new(&prefab.path); - let prefab_content = fs::read_to_string(&path).unwrap(); + let prefab_content = fs::read_to_string(path).unwrap(); let matching_materials: Vec = materials .clone() .into_iter() @@ -104,27 +110,69 @@ impl Unpacker { .filter(|a| prefab_content.contains(&a.guid)) .collect(); if matching_materials.len() != 1 || 1 != matching_models.len() { - continue; + return; } let material = matching_materials.first().unwrap(); - let model = matching_models.first().unwrap(); - if result_models.iter().any(|a| model.guid.eq(&a.guid)) { - continue; - } + let model: &Asset = matching_models.first().unwrap(); let texture_guid: Option = material.try_get_mat_texture_guid(); - let texture_asset: &Asset; - match &texture_guid { + + let texture_asset: &Asset = match &texture_guid { Some(guid) => { - texture_asset = self.assets.iter().find(|a| guid.eq(&a.guid)).unwrap() + self.assets.iter().find(|a| guid.eq(&a.guid)).unwrap() } - None => continue, - } + None => return, + }; // here we should read gltf file and replace material texture with Uri based on texture_asset + let model_path = Path::new(&model.path).with_extension("glb"); + Self::modify_material(&model_path, Path::new(&texture_asset.path)); + }); + } - result_models.push(model.clone()); - counter += 1; + fn align_to_multiple_of_four(n: &mut usize) { + *n = (*n + 3) & !3; + } + + fn modify_material(gltf_path: &Path, texture_asset: &Path) { + let file = fs::File::open(gltf_path).unwrap(); + let reader = io::BufReader::new(file); + let mut gltf = gltf::Gltf::from_reader(reader).unwrap(); + let mut json = gltf.document.into_json(); + if let Some(rel_path) = get_relative_path(texture_asset, gltf_path) { + for image in json.images.iter_mut() { + let result = rel_path.file_name().unwrap().to_str().unwrap().to_string(); + let required_file = gltf_path.with_file_name(&result); + if !required_file.exists() { + fs::copy(texture_asset, gltf_path.with_file_name(&result)).unwrap(); + } + println!( + "Image{:?}: {:?} to be replaced with: {}", + image.name, image.uri, &result + ); + image.uri = Some(result); + } } - println!("Updated {} models", counter); + + gltf.document = Document::from_json(json.clone()).unwrap(); + // Save the modified glTF + let json_string = json::serialize::to_string(&json).expect("Serialization error"); + let mut json_offset = json_string.len(); + Self::align_to_multiple_of_four(&mut json_offset); + let blob = gltf.blob.clone().unwrap_or_default(); + let buffer_length = blob.len(); + let glb = gltf::binary::Glb { + header: gltf::binary::Header { + magic: *b"glTF", + version: 2, + // N.B., the size of binary glTF file is limited to range of `u32`. + length: (json_offset + buffer_length) + .try_into() + .expect("file size exceeds binary glTF limit"), + }, + bin: Some(Cow::Owned(gltf.blob.unwrap_or_default())), + json: Cow::Owned(json_string.into_bytes()), + }; + let writer = std::fs::File::create(gltf_path).expect("I/O error"); + glb.to_writer(writer).expect("glTF binary output error"); } pub fn process_data(&self) { @@ -156,11 +204,11 @@ impl Unpacker { if self.args.fbx_to_gltf.is_some() && &asset.asset_type == &AssetType::FbxModel { process_fbx_file( &source_asset, - &path, + path, &self.args.fbx_to_gltf.clone().unwrap(), ); } else { - process_non_fbx_file(&source_asset, &path); + process_non_fbx_file(&source_asset, path); } }); From c46de6ebae6aff1be8aa090a2b0ba2f47390afc6 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:06:28 +0100 Subject: [PATCH 05/10] Simplify process of FBX model conversion. --- Cargo.toml | 1 + src/asset.rs | 7 ++--- src/unpacker.rs | 82 ++++++++++++++++++++++--------------------------- 3 files changed, 40 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 16d68ba..c9b62c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ opt-level = 2 clap = { version = "4.4", features = ["derive"] } flate2 = "1.0" gltf = "1.4.0" +pathdiff = "0.2.1" rayon = "1.8.0" regex = "1.10.2" tar = "0.4" diff --git a/src/asset.rs b/src/asset.rs index 29a8412..31cadbc 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -6,7 +6,6 @@ use std::fs::File; use std::io::BufRead; use std::io::BufReader; use std::path::Path; -use std::path::PathBuf; #[derive(Clone)] pub struct Asset { @@ -34,9 +33,9 @@ impl Asset { } let file = File::open(&self.path).unwrap(); let buf_reader = BufReader::new(file); - let search = buf_reader.lines().into_iter().find(|s| { + let search = buf_reader.lines().find(|s| { let ss = s.as_ref().unwrap(); - return ss.contains("m_Texture") && ss.contains("guid: "); + ss.contains("m_Texture") && ss.contains("guid: ") }); if let Some(line) = search { let line = line.unwrap_or_default(); @@ -65,7 +64,7 @@ impl Asset { } } - pub fn from_path(entry: &DirEntry, output_dir: &PathBuf) -> Option { + pub fn from_path(entry: &DirEntry, output_dir: &Path) -> Option { let root_file = entry.path(); if !root_file.is_dir() { return None; diff --git a/src/unpacker.rs b/src/unpacker.rs index fa9b25c..aeab852 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -56,7 +56,7 @@ impl Unpacker { .par_bridge() .for_each_with(sender, |s, entry| { let entry = entry.unwrap(); - let asset = crate::asset::Asset::from_path(&entry, &output_dir.to_path_buf()); + let asset = crate::asset::Asset::from_path(&entry, output_dir); if let Some(asset) = asset { let extension = &asset.extension.clone().unwrap_or_default(); if !ignored_extensions.contains(extension) { @@ -75,19 +75,19 @@ impl Unpacker { .assets .clone() .into_iter() - .filter(|a| &a.asset_type == &AssetType::FbxModel) + .filter(|a| a.asset_type == AssetType::FbxModel) .collect(); let prefabs: Vec = self .assets .clone() .into_iter() - .filter(|a| &a.asset_type == &AssetType::Prefab) + .filter(|a| a.asset_type == AssetType::Prefab) .collect(); let materials: Vec = self .assets .clone() .into_iter() - .filter(|a| &a.asset_type == &AssetType::Material) + .filter(|a| a.asset_type == AssetType::Material) .collect(); println!( "There are {} models, {} prefabs and {} materials", @@ -96,7 +96,7 @@ impl Unpacker { materials.len() ); - prefabs.par_iter().for_each(|prefab|{ + prefabs.par_iter().for_each(|prefab| { let path = Path::new(&prefab.path); let prefab_content = fs::read_to_string(path).unwrap(); let matching_materials: Vec = materials @@ -115,11 +115,9 @@ impl Unpacker { let material = matching_materials.first().unwrap(); let model: &Asset = matching_models.first().unwrap(); let texture_guid: Option = material.try_get_mat_texture_guid(); - + let texture_asset: &Asset = match &texture_guid { - Some(guid) => { - self.assets.iter().find(|a| guid.eq(&a.guid)).unwrap() - } + Some(guid) => self.assets.iter().find(|a| guid.eq(&a.guid)).unwrap(), None => return, }; // here we should read gltf file and replace material texture with Uri based on texture_asset @@ -180,11 +178,10 @@ impl Unpacker { let copy_meta_files = self.args.copy_meta_files; let tmp_path = Path::new("./tmp_dir"); - let mapping_arc = Arc::new(&self.assets); let tmp_dir = Arc::new(tmp_path); fs::create_dir(output_dir).unwrap(); - mapping_arc.par_iter().for_each(|asset| { + self.assets.par_iter().for_each(|asset| { let asset_hash = &asset.guid; let path = Path::new(&asset.path); let source_asset = Path::new(&*tmp_dir).join(asset_hash).join("asset"); @@ -201,47 +198,40 @@ impl Unpacker { panic!("SOURCE ASSET DOES NOT EXIST: {}", source_asset.display()); } - if self.args.fbx_to_gltf.is_some() && &asset.asset_type == &AssetType::FbxModel { - process_fbx_file( - &source_asset, - path, - &self.args.fbx_to_gltf.clone().unwrap(), - ); + if self.args.fbx_to_gltf.is_some() && asset.asset_type == AssetType::FbxModel { + self.process_fbx_file(&source_asset, path); } else { - process_non_fbx_file(&source_asset, path); + fs::rename(source_asset, path).unwrap(); } }); fs::remove_dir_all(Path::new(&*tmp_dir)).unwrap(); + } - fn process_fbx_file(source_asset: &Path, result_path: &Path, tool: &PathBuf) { - let out_path = result_path.with_extension(""); - println!( - "{:?}", - &[ - "--input", - source_asset.to_str().unwrap(), - "--output", - out_path.to_str().unwrap() - ] - ); - let output = Command::new(tool) - .args([ - "--input", - source_asset.to_str().unwrap(), - "-b", - "--output", - out_path.to_str().unwrap(), - ]) - .output() - .unwrap(); - let output_result = String::from_utf8_lossy(&output.stdout); - println!("output: {}", output_result); - } - - fn process_non_fbx_file(source_asset: &Path, result_path: &Path) { - fs::rename(source_asset, result_path).unwrap(); - } + fn process_fbx_file(&self, source_asset: &Path, result_path: &Path) { + let tool = self.args.fbx_to_gltf.clone().unwrap(); + let out_path = result_path.with_extension(""); + println!( + "{:?}", + &[ + "--input", + source_asset.to_str().unwrap(), + "--output", + out_path.to_str().unwrap() + ] + ); + let output = Command::new(tool) + .args([ + "--input", + source_asset.to_str().unwrap(), + "-b", + "--output", + out_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + let output_result = String::from_utf8_lossy(&output.stdout); + println!("output: {}", output_result); } fn extract_archive(archive_path: &Path, extract_to: &Path) -> io::Result<()> { From c3b959f63210cd67caf62c8632afd596f2109dbc Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:16:51 +0100 Subject: [PATCH 06/10] New flag: get_materials_from_prefabs --- Cargo.toml | 2 +- src/args.rs | 5 +++++ src/unpacker.rs | 23 ++++++++--------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c9b62c0..2812995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lwa_unity_unpack" -version = "0.3.0" +version = "0.4.0" edition = "2021" repository = "https://github.com/Leinnan/lwa_unity_unpack" homepage = "https://github.com/Leinnan/lwa_unity_unpack" diff --git a/src/args.rs b/src/args.rs index 6de2b6f..19c53fe 100644 --- a/src/args.rs +++ b/src/args.rs @@ -16,6 +16,11 @@ pub struct Args { #[arg(short, long)] pub fbx_to_gltf: Option, + /// checks if material base texture in prefabs differ from the one specified in fbx model + /// that is converted to GLTF and overrides it with the one from prefab and copy texture to models folder + #[arg(long, default_value = "false", default_missing_value = "true")] + pub get_materials_from_prefabs: bool, + /// optional- extensions that will be ignored during unpacking #[arg(long, action = clap::ArgAction::Append)] pub ignore_extensions: Option>, diff --git a/src/unpacker.rs b/src/unpacker.rs index aeab852..6390b9d 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -68,7 +68,7 @@ impl Unpacker { } pub fn update_gltf_materials(&self) { - if self.args.fbx_to_gltf.is_none() { + if self.args.fbx_to_gltf.is_none() || !self.args.get_materials_from_prefabs { return; } let fbx_models: Vec = self @@ -122,7 +122,7 @@ impl Unpacker { }; // here we should read gltf file and replace material texture with Uri based on texture_asset let model_path = Path::new(&model.path).with_extension("glb"); - Self::modify_material(&model_path, Path::new(&texture_asset.path)); + Self::update_material(&model_path, Path::new(&texture_asset.path)); }); } @@ -130,7 +130,7 @@ impl Unpacker { *n = (*n + 3) & !3; } - fn modify_material(gltf_path: &Path, texture_asset: &Path) { + fn update_material(gltf_path: &Path, texture_asset: &Path) { let file = fs::File::open(gltf_path).unwrap(); let reader = io::BufReader::new(file); let mut gltf = gltf::Gltf::from_reader(reader).unwrap(); @@ -211,16 +211,7 @@ impl Unpacker { fn process_fbx_file(&self, source_asset: &Path, result_path: &Path) { let tool = self.args.fbx_to_gltf.clone().unwrap(); let out_path = result_path.with_extension(""); - println!( - "{:?}", - &[ - "--input", - source_asset.to_str().unwrap(), - "--output", - out_path.to_str().unwrap() - ] - ); - let output = Command::new(tool) + let _output = Command::new(tool) .args([ "--input", source_asset.to_str().unwrap(), @@ -230,8 +221,10 @@ impl Unpacker { ]) .output() .unwrap(); - let output_result = String::from_utf8_lossy(&output.stdout); - println!("output: {}", output_result); + println!( + "Fbx converted to GLTF: {}", + out_path.with_extension("glb").to_str().unwrap() + ); } fn extract_archive(archive_path: &Path, extract_to: &Path) -> io::Result<()> { From 1d7e43fa5ca4bf40d535599330c06b497d7f34ed Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:20:28 +0100 Subject: [PATCH 07/10] Update changelog, README --- CHANGELOG.md | 6 ++++++ Cargo.lock | 2 +- README.md | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 126ba46..40dfa26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.4.0] + +### Added + +- New flag `--get_materials_from_prefabs` for updating model textures based on prefabs and materials. + ## [0.3.0] ### Added diff --git a/Cargo.lock b/Cargo.lock index 96edb54..004eefa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,7 +345,7 @@ checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lwa_unity_unpack" -version = "0.3.0" +version = "0.4.0" dependencies = [ "clap", "flate2", diff --git a/README.md b/README.md index f2b205a..aa689fc 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Options: -i, --input .unitypackage file to extract -o, --output target directory -f, --fbx-to-gltf optional- path to the tool that will auto convert fbx files to gltf during unpacking + --get-materials-from-prefabs + checks if material base texture in prefabs differ from the one specified in fbx model that is converted to GLTF and overrides it with the one from prefab and copy texture to models folder --ignore-extensions optional- extensions that will be ignored during unpacking --copy-meta-files From 6a19b0fbb091ed48bc1b8ff9539e6608af4d9f93 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:25:23 +0100 Subject: [PATCH 08/10] Remove pathdiff dependency and unused function --- Cargo.toml | 1 - src/unpacker.rs | 33 ++++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2812995..644380d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ opt-level = 2 clap = { version = "4.4", features = ["derive"] } flate2 = "1.0" gltf = "1.4.0" -pathdiff = "0.2.1" rayon = "1.8.0" regex = "1.10.2" tar = "0.4" diff --git a/src/unpacker.rs b/src/unpacker.rs index 6390b9d..b449ce7 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -5,7 +5,7 @@ use rayon::prelude::*; use std::borrow::Cow; use std::fs::File; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use std::sync::mpsc::channel; use std::sync::Arc; @@ -18,10 +18,6 @@ pub struct Unpacker { pub assets: Vec, } -fn get_relative_path(path1: &Path, path2: &Path) -> Option { - pathdiff::diff_paths(path1, path2) -} - impl Unpacker { pub fn prepare_environment(&self) { let archive_path = Path::new(&self.args.input); @@ -135,19 +131,22 @@ impl Unpacker { let reader = io::BufReader::new(file); let mut gltf = gltf::Gltf::from_reader(reader).unwrap(); let mut json = gltf.document.into_json(); - if let Some(rel_path) = get_relative_path(texture_asset, gltf_path) { - for image in json.images.iter_mut() { - let result = rel_path.file_name().unwrap().to_str().unwrap().to_string(); - let required_file = gltf_path.with_file_name(&result); - if !required_file.exists() { - fs::copy(texture_asset, gltf_path.with_file_name(&result)).unwrap(); - } - println!( - "Image{:?}: {:?} to be replaced with: {}", - image.name, image.uri, &result - ); - image.uri = Some(result); + for image in json.images.iter_mut() { + let result = texture_asset.file_name().unwrap().to_str().unwrap().to_string(); + let required_file = gltf_path.with_file_name(&result); + if !required_file.exists() { + fs::copy(texture_asset, gltf_path.with_file_name(&result)).unwrap(); } + if let Some(old_path) = &image.uri { + if old_path.eq(&result){ + return; + } + } + println!( + "Image{:?}: {:?} to be replaced with: {}", + image.name, image.uri, &result + ); + image.uri = Some(result); } gltf.document = Document::from_json(json.clone()).unwrap(); From 760dded5eb5f1dd20a30477b9c485205a2cf09e8 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:28:49 +0100 Subject: [PATCH 09/10] Refactor unpacker.rs- asset filtering --- Cargo.lock | 7 ------- src/unpacker.rs | 39 +++++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 004eefa..f2291cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,7 +350,6 @@ dependencies = [ "clap", "flate2", "gltf", - "pathdiff", "rayon", "regex", "tar", @@ -411,12 +410,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - [[package]] name = "png" version = "0.17.10" diff --git a/src/unpacker.rs b/src/unpacker.rs index b449ce7..95f6f16 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -35,6 +35,7 @@ impl Unpacker { fs::remove_dir_all(output_dir).unwrap(); } } + pub fn extract(&mut self) { let archive_path = Path::new(&self.args.input); let tmp_path = Path::new("./tmp_dir"); @@ -63,28 +64,21 @@ impl Unpacker { self.assets = receiver.iter().collect(); } + pub fn assets_of_type(&self, asset_type: AssetType) -> Vec { + self.assets + .clone() + .into_iter() + .filter(|a| a.asset_type == asset_type) + .collect() + } + pub fn update_gltf_materials(&self) { if self.args.fbx_to_gltf.is_none() || !self.args.get_materials_from_prefabs { return; } - let fbx_models: Vec = self - .assets - .clone() - .into_iter() - .filter(|a| a.asset_type == AssetType::FbxModel) - .collect(); - let prefabs: Vec = self - .assets - .clone() - .into_iter() - .filter(|a| a.asset_type == AssetType::Prefab) - .collect(); - let materials: Vec = self - .assets - .clone() - .into_iter() - .filter(|a| a.asset_type == AssetType::Material) - .collect(); + let fbx_models = self.assets_of_type(AssetType::FbxModel); + let prefabs = self.assets_of_type(AssetType::Prefab); + let materials = self.assets_of_type(AssetType::Material); println!( "There are {} models, {} prefabs and {} materials", fbx_models.len(), @@ -132,13 +126,18 @@ impl Unpacker { let mut gltf = gltf::Gltf::from_reader(reader).unwrap(); let mut json = gltf.document.into_json(); for image in json.images.iter_mut() { - let result = texture_asset.file_name().unwrap().to_str().unwrap().to_string(); + let result = texture_asset + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); let required_file = gltf_path.with_file_name(&result); if !required_file.exists() { fs::copy(texture_asset, gltf_path.with_file_name(&result)).unwrap(); } if let Some(old_path) = &image.uri { - if old_path.eq(&result){ + if old_path.eq(&result) { return; } } From 6483ebfc56e738b90ac6cc4df8814b0f8bb96868 Mon Sep 17 00:00:00 2001 From: Piotr Siuszko Date: Fri, 29 Dec 2023 15:30:30 +0100 Subject: [PATCH 10/10] Update deps --- Cargo.lock | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2291cd..af2aed2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,9 +109,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -119,9 +119,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -181,21 +181,20 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if", ] @@ -357,18 +356,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "miniz_oxide" @@ -425,9 +415,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" dependencies = [ "unicode-ident", ] @@ -563,9 +553,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.41" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -669,9 +659,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "xattr" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", "linux-raw-sys",