Add rayon for parallel processing in asset mapping
This commit is contained in:
parent
4c182f8f67
commit
c1e0e83206
|
|
@ -56,6 +56,12 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
|
|
@ -116,6 +122,44 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.8"
|
||||
|
|
@ -166,9 +210,19 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"clap",
|
||||
"flate2",
|
||||
"rayon",
|
||||
"tar",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.1"
|
||||
|
|
@ -178,6 +232,26 @@ dependencies = [
|
|||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
|
|
|
|||
|
|
@ -8,4 +8,5 @@ edition = "2021"
|
|||
[dependencies]
|
||||
clap = "4.4"
|
||||
flate2 = "1.0"
|
||||
rayon = "1.8.0"
|
||||
tar = "0.4"
|
||||
|
|
|
|||
28
src/main.rs
28
src/main.rs
|
|
@ -1,5 +1,5 @@
|
|||
use std::fs::File;
|
||||
use std::{fs, io};
|
||||
use std::{fs, io, sync::Arc};
|
||||
use std::path::Path;
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
|
@ -9,6 +9,7 @@ use std::hash::Hash;
|
|||
use std::io::prelude::*;
|
||||
use std::io::BufReader;
|
||||
use std::process::Command;
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub fn extract_archive(archive_path: &Path, extract_to: &Path) -> io::Result<()> {
|
||||
let tar_gz = File::open(archive_path)?;
|
||||
|
|
@ -40,21 +41,17 @@ fn main() {
|
|||
let entry = entry.unwrap();
|
||||
let root_file = entry.path();
|
||||
let asset = entry.file_name().into_string().unwrap();
|
||||
|
||||
if root_file.is_dir() {
|
||||
let mut real_path = String::new();
|
||||
let mut has_asset = false;
|
||||
|
||||
for sub_entry in fs::read_dir(root_file.clone()).unwrap() {
|
||||
let sub_entry = sub_entry.unwrap();
|
||||
let file_name = sub_entry.file_name().into_string().unwrap();
|
||||
|
||||
if file_name == "pathname" {
|
||||
let path = sub_entry.path();
|
||||
let file = File::open(path).unwrap();
|
||||
let mut buf_reader = BufReader::new(file);
|
||||
let line = buf_reader.lines().next();
|
||||
|
||||
match line {
|
||||
Some(Ok(path)) => real_path = path,
|
||||
_ => continue,
|
||||
|
|
@ -63,17 +60,19 @@ fn main() {
|
|||
has_asset = true;
|
||||
}
|
||||
}
|
||||
|
||||
if has_asset {
|
||||
mapping.insert(asset, real_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Results:");
|
||||
for (asset_hash, asset_path) in &mapping {
|
||||
let mut mapping_arc = Arc::new(mapping);
|
||||
let tmp_dir = Arc::new(tmp_dir.clone());
|
||||
let output_dir = Arc::new(output_dir.clone());
|
||||
|
||||
mapping_arc.par_iter().for_each(|(asset_hash, asset_path)| {
|
||||
let path = Path::new(asset_path);
|
||||
let source_asset = Path::new(tmp_dir).join(asset_hash).join("asset");
|
||||
let source_asset = Path::new(&*tmp_dir).join(asset_hash).join("asset");
|
||||
let result_path = output_dir.join(&path);
|
||||
|
||||
process_directory(asset_hash, asset_path, &result_path);
|
||||
|
|
@ -81,12 +80,13 @@ fn main() {
|
|||
|
||||
if let Some("fbx") = path.extension().and_then(OsStr::to_str) {
|
||||
process_fbx_file(&source_asset, &result_path);
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
process_non_fbx_file(&source_asset, &result_path);
|
||||
}
|
||||
fs::remove_dir_all(tmp_dir).unwrap();
|
||||
});
|
||||
|
||||
fs::remove_dir_all(Path::new(&*tmp_dir)).unwrap();
|
||||
|
||||
fn process_directory(asset_hash: &str, asset_path: &str, result_path: &Path) {
|
||||
println!("{}: {:?}", asset_hash, asset_path);
|
||||
|
|
@ -115,8 +115,4 @@ fn main() {
|
|||
fn process_non_fbx_file(source_asset: &Path, result_path: &Path) {
|
||||
fs::rename(source_asset, result_path).unwrap();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue