diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index 84f60a3098e..a23ab8b23fc 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -4,7 +4,7 @@ name: CICD # spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS CLEVEL nodocs # spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata # spell-checker:ignore (people) Peltoche rivy Anson dawidd -# spell-checker:ignore (shell/tools) binutils choco clippy dmake esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libfuse libssl limactl nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs zstd +# spell-checker:ignore (shell/tools) binutils choco clippy dmake esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libcrypto libfuse libssl limactl nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs zstd # spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nofeatures nullglob onexitbegin onexitend pell runtest tempfile testsuite toybox uutils libsystemd codspeed wasip libexecinfo env: @@ -827,6 +827,76 @@ jobs: flags: coverage,${{ matrix.job.os }} fail_ci_if_error: false + test_openssl: + name: Build/OpenSSL + needs: [ min_version, deps ] + runs-on: ${{ matrix.job.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job: + # Linux: exercises both vendored (static) and OPENSSL_NO_VENDOR=1 + # (dynamic against system libcrypto). ubuntu-latest ships libssl-dev. + - { os: ubuntu-latest, features: feat_os_unix, dynamic: true } + # macOS: vendored only — system libcrypto needs OPENSSL_DIR + # pointing at Homebrew, which isn't worth wiring up for a smoke test. + - { os: macos-latest, features: feat_os_unix, dynamic: false } + # Windows is omitted: the `openssl` dep is `cfg(unix)`, so the + # feature is a no-op there and there's nothing to exercise. + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@nextest + - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache + id: sccache-setup + uses: mozilla-actions/sccache-action@v0.0.10 + continue-on-error: true + - name: Export sccache + if: steps.sccache-setup.outcome == 'success' + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + # Default mode: `openssl` crate's `vendored` feature builds libcrypto + # from source and statically links it (mirrors how `expr` links + # oniguruma). Runs the md5sum/sha*sum/cksum integration tests end-to-end + # so we'd catch a wrong-digest regression, not just a type error. + - name: Test checksum utilities with OpenSSL (vendored / static) + shell: bash + env: + RUST_BACKTRACE: "1" + run: | + cargo nextest run --hide-progress-bar --profile ci \ + --features "${{ matrix.job.features }},openssl" \ + -E 'test(/^test_(md5sum|sha1sum|sha224sum|sha256sum|sha384sum|sha512sum|cksum)::/)' + # Confirm the vendored build is actually statically linked. If this + # regresses, the `vendored` feature has stopped doing its job. + - name: Verify static linkage (vendored) + if: matrix.job.os == 'ubuntu-latest' + shell: bash + run: | + cargo build --release --features "${{ matrix.job.features }},openssl" --bin coreutils + if ldd target/release/coreutils 2>&1 | grep -iE 'libssl|libcrypto'; then + echo "ERROR: coreutils dynamically links libssl/libcrypto despite vendored feature" + exit 1 + fi + echo "OK: no dynamic libssl/libcrypto linkage" + # Second linking mode: OPENSSL_NO_VENDOR=1 links dynamically against + # the system libcrypto/libssl. + - name: Test checksum utilities with OpenSSL (system / dynamic) + if: matrix.job.dynamic + shell: bash + env: + OPENSSL_NO_VENDOR: "1" + RUST_BACKTRACE: "1" + run: | + cargo nextest run --hide-progress-bar --profile ci \ + --features "${{ matrix.job.features }},openssl" \ + -E 'test(/^test_(md5sum|sha1sum|sha224sum|sha256sum|sha384sum|sha512sum|cksum)::/)' + test_selinux: name: Build/SELinux needs: [ min_version, deps ] diff --git a/Cargo.lock b/Cargo.lock index 419861ef220..3251e2fe5c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1115,6 +1115,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2138,6 +2153,53 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-src" +version = "300.5.5+3.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -4490,6 +4552,7 @@ dependencies = [ "memchr", "nix", "num-traits", + "openssl", "os_display", "procfs", "rustc-hash", @@ -4558,6 +4621,12 @@ dependencies = [ "ansi-width", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 3c63028fc4d..75323d4e65b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ # coreutils (uutils) # * see the repository LICENSE, README, and CONTRIBUTING files for more information -# spell-checker:ignore (libs) bigdecimal datetime foldhash serde gethostid kqueue libselinux mangen memmap uuhelp startswith constness expl unnested logind cfgs interner getauxval +# spell-checker:ignore (libs) bigdecimal datetime foldhash serde gethostid kqueue libcrypto libselinux mangen memmap uuhelp startswith constness expl unnested logind cfgs interner getauxval [package] name = "coreutils" @@ -26,6 +26,31 @@ unix = ["feat_os_unix"] windows = ["feat_os_windows"] ## project-specific feature shortcodes expensive_tests = [] +# Opt-in: use OpenSSL (libcrypto) for md5/sha1/sha2-family digests in the +# checksum utilities instead of the pure-Rust crates. Provides large speedups +# on CPUs without SHA-NI. +# +# Enable with: `cargo build --release --features unix,openssl` +# +# By default OpenSSL is built from source and statically linked (via the +# `vendored` feature of the `openssl` crate), mirroring how `expr` links +# oniguruma. To link against the system libcrypto/libssl dynamically instead, +# set `OPENSSL_NO_VENDOR=1` at build time. +# +# The `?/openssl` entries propagate the choice into each standalone checksum +# crate only when that crate is itself enabled, so disabling this feature +# genuinely turns OpenSSL off everywhere. On Windows the feature is a no-op +# and the pure-Rust implementations are used regardless. +openssl = [ + "cksum?/openssl", + "md5sum?/openssl", + "sha1sum?/openssl", + "sha224sum?/openssl", + "sha256sum?/openssl", + "sha384sum?/openssl", + "sha512sum?/openssl", + "uucore/openssl", +] # "test_risky_names" == enable tests that create problematic file names (would make a network share inaccessible to Windows, breaks SVN on Mac OS, etc.) test_risky_names = [] # * only build `uudoc` when `--feature uudoc` is activated @@ -483,6 +508,7 @@ blake3 = "1.5.1" sm3 = "0.5.0" crc-fast = { version = "1.5.0", default-features = false } digest = "0.11.0" +openssl = { version = "0.10", features = ["vendored"] } # Fluent dependencies fluent = "0.17.0" diff --git a/README.md b/README.md index 9b1080f0007..56dcb22a6e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ - +
@@ -126,6 +126,22 @@ and `libclang` are installed on your system. Then, run the following command: cargo build --release --features unix,feat_selinux ``` +To speed up the checksum utilities (`md5sum`, `sha1sum`, `sha224sum`, `sha256sum`, +`sha384sum`, `sha512sum`, and `cksum`) by using OpenSSL's `libcrypto` instead of +the pure-Rust digest crates, enable the `openssl` feature: +``` +cargo build --release --features unix,openssl +``` +By default OpenSSL is built from source and statically linked into the +binary (mirroring how `expr` links `oniguruma`), so no runtime dependency +on system libcrypto/libssl is added. To link dynamically against the system +libcrypto instead, set `OPENSSL_NO_VENDOR=1` at build time. + +The speedup is largest on CPUs without SHA-NI hardware acceleration. The +feature is a no-op on Windows (the pure-Rust implementations are always used +there) and is automatically bypassed at runtime for any algorithm libcrypto +refuses (for example, MD5 in strict FIPS mode). + If you don't want to build every utility available on your platform into the final binary, you can also specify which ones you want to build manually. For example: diff --git a/build.rs b/build.rs index b2d4c52279d..2b61b84d2e4 100644 --- a/build.rs +++ b/build.rs @@ -49,7 +49,7 @@ pub fn main() { continue; } "default" | "macos" | "unix" | "windows" | "selinux" | "zip" | "clap_complete" - | "clap_mangen" | "fluent_syntax" => continue, // common/standard feature names + | "clap_mangen" | "fluent_syntax" | "openssl" => continue, // common/standard feature names "nightly" | "test_unimplemented" | "expensive_tests" | "test_risky_names" => { continue; } // crate-local custom features diff --git a/src/uu/cksum/Cargo.toml b/src/uu/cksum/Cargo.toml index 73c6f962128..4081bbf5534 100644 --- a/src/uu/cksum/Cargo.toml +++ b/src/uu/cksum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ uu_checksum_common = { workspace = true } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] divan = { workspace = true } uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/md5sum/Cargo.toml b/src/uu/md5sum/Cargo.toml index b0bdd50ee2a..8a071ff2989 100644 --- a/src/uu/md5sum/Cargo.toml +++ b/src/uu/md5sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/sha1sum/Cargo.toml b/src/uu/sha1sum/Cargo.toml index 19ae1a1e463..bc265752872 100644 --- a/src/uu/sha1sum/Cargo.toml +++ b/src/uu/sha1sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/sha224sum/Cargo.toml b/src/uu/sha224sum/Cargo.toml index 6dea3d5bfce..be0362f3f3f 100644 --- a/src/uu/sha224sum/Cargo.toml +++ b/src/uu/sha224sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/sha256sum/Cargo.toml b/src/uu/sha256sum/Cargo.toml index dfeee877cfb..1d2c167c833 100644 --- a/src/uu/sha256sum/Cargo.toml +++ b/src/uu/sha256sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/sha384sum/Cargo.toml b/src/uu/sha384sum/Cargo.toml index 22a4ba397d3..52ac45c21b5 100644 --- a/src/uu/sha384sum/Cargo.toml +++ b/src/uu/sha384sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uu/sha512sum/Cargo.toml b/src/uu/sha512sum/Cargo.toml index fd315af608d..df5e60ebc6b 100644 --- a/src/uu/sha512sum/Cargo.toml +++ b/src/uu/sha512sum/Cargo.toml @@ -30,6 +30,9 @@ uucore = { workspace = true, features = [ ] } fluent = { workspace = true } +[features] +openssl = ["uucore/openssl"] + [dev-dependencies] uucore = { workspace = true, features = ["benchmark"] } diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index 37a14830b4d..5d71efba51e 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -1,4 +1,4 @@ -# spell-checker:ignore (features) bigdecimal zerocopy extendedbigdecimal tzdb zoneinfo logind +# spell-checker:ignore (features) bigdecimal zerocopy extendedbigdecimal tzdb zoneinfo logind libcrypto [package] name = "uucore" @@ -108,6 +108,9 @@ nix = { workspace = true, features = [ "user", "zerocopy", ] } +# OpenSSL is only used on Unix targets. On Windows, enabling the `openssl` +# feature is a no-op and the pure-Rust digest implementations are used. +openssl = { workspace = true, optional = true } walkdir = { workspace = true, optional = true } xattr = { workspace = true, optional = true } @@ -199,6 +202,10 @@ sum = [ "crc-fast", "data-encoding", ] +# Use OpenSSL (libcrypto) for md5/sha1/sha2-family digests instead of the +# pure-Rust crates. This dramatically speeds up checksum utilities on CPUs +# without SHA-NI by using OpenSSL's optimized assembly implementations. +openssl = ["dep:openssl"] update-control = ["parser"] utf8 = [] utmpx = ["time", "time/macros", "libc", "dns-lookup"] diff --git a/src/uucore/src/lib/features/sum.rs b/src/uucore/src/lib/features/sum.rs index 85a3d9f58c0..e11ad4fb042 100644 --- a/src/uucore/src/lib/features/sum.rs +++ b/src/uucore/src/lib/features/sum.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore memmem algo PCLMULQDQ refin xorout Hdlc +// spell-checker:ignore memmem algo PCLMULQDQ refin xorout Hdlc libcrypto SSSE FIPS //! Implementations of digest functions, like md5 and sha1. //! @@ -451,19 +451,131 @@ macro_rules! impl_digest_shake { }; } +// When the `openssl` feature is enabled, md5/sha1/sha2-family digests are +// backed by OpenSSL's libcrypto, which provides hand-tuned assembly +// implementations (AVX2, SSSE3, etc.) that are substantially faster than the +// pure-Rust crates on CPUs without SHA-NI. The OpenSSL backend is only +// compiled in on `cfg(unix)`; Windows targets always use the pure-Rust path +// since libcrypto headers aren't generally available there. +#[cfg(not(all(feature = "openssl", unix)))] pub struct Md5(md5::Md5); +#[cfg(not(all(feature = "openssl", unix)))] pub struct Sha1(sha1::Sha1); +#[cfg(not(all(feature = "openssl", unix)))] pub struct Sha224(sha2::Sha224); +#[cfg(not(all(feature = "openssl", unix)))] pub struct Sha256(sha2::Sha256); +#[cfg(not(all(feature = "openssl", unix)))] pub struct Sha384(sha2::Sha384); +#[cfg(not(all(feature = "openssl", unix)))] pub struct Sha512(sha2::Sha512); + +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Md5, 128); +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Sha1, 160); +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Sha224, 224); +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Sha256, 256); +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Sha384, 384); +#[cfg(not(all(feature = "openssl", unix)))] impl_digest_common!(Sha512, 512); +// When OpenSSL is built in FIPS mode (or otherwise refuses an algorithm — +// e.g. MD5/SHA-1 in strict legacy-off builds), `Hasher::new` returns Err. +// To avoid panicking at construction time, each type carries a PureRust +// fallback variant built from the same crate the non-OpenSSL path uses. +#[cfg(all(feature = "openssl", unix))] +macro_rules! impl_digest_openssl { + ($algo_type:ident, $size:literal, $md:expr, $rust_type:ty) => { + pub enum $algo_type { + OpenSsl(openssl::hash::Hasher), + PureRust($rust_type), + } + + impl $algo_type { + pub const BIT_SIZE: usize = $size; + } + + impl Default for $algo_type { + fn default() -> Self { + match openssl::hash::Hasher::new($md) { + Ok(h) => Self::OpenSsl(h), + Err(_) => Self::PureRust(<$rust_type>::default()), + } + } + } + + impl Digest for $algo_type { + fn hash_update(&mut self, input: &[u8]) { + match self { + Self::OpenSsl(h) => { + h.update(input).expect("OpenSSL hasher update failed"); + } + Self::PureRust(h) => digest::Digest::update(h, input), + } + } + + fn hash_finalize(&mut self, out: &mut [u8]) { + match self { + // `finish` finalizes the hash and resets the underlying context. + Self::OpenSsl(h) => { + let result = h.finish().expect("OpenSSL hasher finish failed"); + out.copy_from_slice(&result); + } + Self::PureRust(h) => { + let result = digest::Digest::finalize_reset(h); + out.copy_from_slice(&result); + } + } + } + + fn reset(&mut self) { + *self = Self::default(); + } + + fn output_bits(&self) -> usize { + Self::BIT_SIZE + } + } + }; +} + +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!(Md5, 128, openssl::hash::MessageDigest::md5(), md5::Md5); +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!(Sha1, 160, openssl::hash::MessageDigest::sha1(), sha1::Sha1); +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!( + Sha224, + 224, + openssl::hash::MessageDigest::sha224(), + sha2::Sha224 +); +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!( + Sha256, + 256, + openssl::hash::MessageDigest::sha256(), + sha2::Sha256 +); +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!( + Sha384, + 384, + openssl::hash::MessageDigest::sha384(), + sha2::Sha384 +); +#[cfg(all(feature = "openssl", unix))] +impl_digest_openssl!( + Sha512, + 512, + openssl::hash::MessageDigest::sha512(), + sha2::Sha512 +); + pub struct Sha3_224(sha3::Sha3_224); pub struct Sha3_256(sha3::Sha3_256); pub struct Sha3_384(sha3::Sha3_384);