diff --git a/Cargo.lock b/Cargo.lock index 2fe5bce..7cb7431 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" [[package]] name = "apple-bindgen" @@ -341,25 +341,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel 2.3.1", - "async-io 2.4.0", - "async-lock 3.4.0", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.3.1", - "futures-lite 2.5.0", - "rustix 0.38.41", - "tracing", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -391,20 +372,20 @@ dependencies = [ [[package]] name = "async-std" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c634475f29802fde2b8f0b505b1bd00dfe4df7d4a000f0b36f7671197d5c3615" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ "async-channel 1.9.0", "async-global-executor", - "async-io 2.4.0", - "async-lock 3.4.0", - "async-process 2.3.0", + "async-io 1.13.0", + "async-lock 2.8.0", + "async-process", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite 2.5.0", + "futures-lite 1.13.0", "gloo-timers", "kv-log-macro", "log", @@ -491,9 +472,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "batch-channel" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceabc1a50ae0876db2752190f2846ba3afa2ba736d1974aa643c15178afa42f3" +checksum = "56fcd71a62667be5ee74fc3c6515364ecf278632d0d6801cd4022ec1e14717db" dependencies = [ "futures-core", "pin-project", @@ -550,15 +531,16 @@ checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" [[package]] name = "blake3" -version = "1.5.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +checksum = "dcd555c66291d5f836dbb6883b48660ece810fe25a31f3bdfb911945dff2691f" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "digest 0.9.0", ] [[package]] @@ -718,9 +700,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -728,9 +710,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -788,9 +770,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "convert_case" @@ -1039,6 +1021,15 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.7" @@ -1051,9 +1042,9 @@ dependencies = [ [[package]] name = "directories" -version = "5.0.1" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +checksum = "74be3be809c18e089de43bdc504652bb2bc473fca8756131f8689db8cf079ba9" dependencies = [ "dirs-sys", ] @@ -1243,9 +1234,9 @@ checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "90fa4cc29d25b0687b8570b0da86eac698dcb525110ad8b938fe6712baa711ec" dependencies = [ "futures-channel", "futures-core", @@ -1416,9 +1407,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "gloo-timers" -version = "0.3.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" dependencies = [ "futures-channel", "futures-core", @@ -1530,9 +1521,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "image" -version = "0.25.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" dependencies = [ "bytemuck", "byteorder-lite", @@ -1553,9 +1544,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.0" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f" +checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" dependencies = [ "byteorder-lite", "quick-error", @@ -1932,7 +1923,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" dependencies = [ "cfg-if", - "rayon", ] [[package]] @@ -1942,7 +1932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -2118,6 +2108,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + [[package]] name = "objc" version = "0.2.7" @@ -2200,7 +2200,7 @@ dependencies = [ "chrono", "clap", "criterion", - "digest", + "digest 0.10.7", "directories", "dunce", "futures", @@ -2223,7 +2223,7 @@ dependencies = [ "sha1", "sha2", "static_assertions", - "thiserror 2.0.4", + "thiserror 2.0.0", "tokio", "turbojpeg", "vcpkg", @@ -2543,7 +2543,6 @@ dependencies = [ "loop9", "quick-error", "rav1e", - "rayon", "rgb", ] @@ -2627,6 +2626,9 @@ name = "rgb" version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" +dependencies = [ + "bytemuck", +] [[package]] name = "rusqlite" @@ -2644,9 +2646,9 @@ dependencies = [ [[package]] name = "rusqlite_migration" -version = "1.3.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923b42e802f7dc20a0a6b5e097ba7c83fe4289da07e49156fecf6af08aa9cd1c" +checksum = "6f44e7b609eaab5d2232bd2de69f31101b380d4161e9de998a743cef21633679" dependencies = [ "log", "rusqlite", @@ -2753,9 +2755,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "self_cell" -version = "1.1.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe" +checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" [[package]] name = "semver" @@ -2823,7 +2825,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -2834,7 +2836,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -2999,11 +3001,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "15291287e9bff1bc6f9ff3409ed9af665bec7a5fc8ac079ea96be07bca0e2668" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.0", ] [[package]] @@ -3019,9 +3021,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "22efd00f33f93fa62848a7cab956c3d38c8d43095efda1decfc2b3a5dc0b8972" dependencies = [ "proc-macro2", "quote", @@ -3092,20 +3094,21 @@ dependencies = [ [[package]] name = "tokio" -version = "1.42.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", + "num_cpus", "pin-project-lite", "tokio-macros", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -3792,7 +3795,7 @@ dependencies = [ "async-fs", "async-io 1.13.0", "async-lock 2.8.0", - "async-process 1.8.1", + "async-process", "async-recursion", "async-task", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index fec075b..38d6f72 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,10 +37,6 @@ vcpkg = "0.2.15" [dev-dependencies] criterion = { version = "0.5.1", features = ["html_reports"] } -digest = "0.10.7" -md-5 = "0.10.6" -sha1 = "0.10.6" -sha2 = "0.10.8" [[bench]] name = "databasebench" @@ -54,10 +50,9 @@ blake3 = "1" blockhash = "1.0.0" chrono = "0.4.38" clap = { version = "4.5.20", features = ["derive"] } +digest = "0.10.7" directories = "5" dunce = "1.0.5" -kamadak-exif = "0.6.1" -keepawake = "0.5.1" futures = "0.3" hex = "0.4.3" image = { version = "0.25.2", features = [] } @@ -65,11 +60,16 @@ image_hasher = "2.0.0" indoc = "2.0.5" itermore = { version = "0.7.1", features = ["array_chunks"] } jwalk = "0.8.1" -ntapi = "0.4.1" +kamadak-exif = "0.6.1" +keepawake = "0.5.1" libheif-rs = { version = "1.0.2", default-features = false } +md-5 = "0.10.6" +ntapi = "0.4.1" rayon = "1.10.0" rusqlite_migration = "1.3.0" self_cell = "1.0.4" +sha1 = "0.10.6" +sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "2.0.0" tokio = { version = "1.38", features = ["macros", "rt-multi-thread", "sync"] } diff --git a/src/main.rs b/src/main.rs index b5348fd..5ea8dd2 100755 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,9 @@ #![allow(clippy::redundant_pattern_matching)] use clap::Parser; +use digest::DynDigest; use photohash::iopool; -use photohash::model::Hash32; +use photohash::model::{ExtraHashes, Hash32}; use std::io::Read; use std::path::PathBuf; @@ -33,6 +34,44 @@ async fn compute_blake3(path: PathBuf) -> anyhow::Result { .await } +async fn compute_extra_hashes(path: PathBuf) -> anyhow::Result { + // TODO: To save IO, this could be folded into compute_blake3 on + // the initial computation. + + iopool::run_in_io_pool(move || { + let mut extra_hashes = ExtraHashes::default(); + + let mut hash_md5 = md5::Md5::default(); + let mut hash_sha1 = sha1::Sha1::default(); + let mut hash_sha256 = sha2::Sha256::default(); + let mut hashers: [(&mut dyn DynDigest, &mut [u8]); 3] = [ + (&mut hash_md5, extra_hashes.md5.get_or_insert_default()), + (&mut hash_sha1, extra_hashes.sha1.get_or_insert_default()), + (&mut hash_sha256, extra_hashes.sha256.get_or_insert_default()), + ]; + let mut file = std::fs::File::open(path)?; + let mut buffer = [0u8; READ_SIZE]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + let buffer = &buffer[..n]; + for (h, _) in &mut hashers { + h.update(buffer); + } + } + + + for (h, dest) in &mut hashers { + h.finalize_into_reset(dest).expect("invalid buffer size"); + } + + Ok(extra_hashes) + }) + .await +} + #[derive(Parser)] #[command(name = "photohash", about = "Index your files", version)] struct Args {