test: close coverage gaps and harden e2e strategy

.github/workflows/integration.yml

@@ -1,28 +1,96 @@
 name: Integration Tests
 
 on:
-  push:
-    branches: [ main ]
   pull_request:
-    branches: [ main ]
-  workflow_dispatch:  # Allow manual triggering
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  TEST_S3_ENDPOINT: http://localhost:9000
+  TEST_S3_ACCESS_KEY: accesskey
+  TEST_S3_SECRET_KEY: secretkey
 
 jobs:
-  integration:
-    name: Integration Tests
+  smoke-latest:
+    name: Smoke (RustFS latest)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Start RustFS latest
+        run: |
+          docker run -d --name rustfs \
+            -p 9000:9000 \
+            -p 9001:9001 \
+            -v rustfs-data:/data \
+            -e RUSTFS_ROOT_USER=accesskey \
+            -e RUSTFS_ROOT_PASSWORD=secretkey \
+            -e RUSTFS_ACCESS_KEY=accesskey \
+            -e RUSTFS_SECRET_KEY=secretkey \
+            -e RUSTFS_VOLUMES=/data \
+            -e RUSTFS_ADDRESS=":9000" \
+            -e RUSTFS_CONSOLE_ENABLE="true" \
+            -e RUSTFS_CONSOLE_ADDRESS=":9001" \
+            rustfs/rustfs:latest
+
+      - name: Wait for RustFS
+        run: |
+          for i in {1..60}; do
+            if curl -sf http://localhost:9000/health > /dev/null 2>&1; then
+              echo "RustFS is ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "RustFS failed to start"
+          docker logs rustfs
+          exit 1
+
+      - name: Run smoke compatibility tests
+        run: |
+          set -euo pipefail
+          TESTS=(
+            "object_operations::test_upload_and_download_small_file"
+            "object_operations::test_move_recursive_prefix_s3_to_s3"
+            "quota_operations::test_bucket_quota_set_info_clear"
+          )
+
+          for test_name in "${TESTS[@]}"; do
+            echo "==> Running $test_name"
+            cargo test \
+              --package rustfs-cli \
+              --test integration \
+              --features integration \
+              "$test_name" \
+              -- \
+              --exact \
+              --test-threads=1
+          done
+
+      - name: Show RustFS logs on failure
+        if: failure()
+        run: docker logs rustfs 2>&1 | tail -200
+
+  full-latest:
+    name: Full Integration (RustFS latest)
     runs-on: ubuntu-latest
-    # Don't block PR merges - integration tests are supplementary
-    continue-on-error: true
+    timeout-minutes: 90
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
 
-      - name: Start RustFS
+      - name: Start RustFS latest
         run: |
           docker run -d --name rustfs \
             -p 9000:9000 \
@@ -36,41 +104,32 @@ jobs:
             -e RUSTFS_ADDRESS=":9000" \
             -e RUSTFS_CONSOLE_ENABLE="true" \
             -e RUSTFS_CONSOLE_ADDRESS=":9001" \
-            rustfs/rustfs:1.0.0-alpha.81
+            rustfs/rustfs:latest
 
       - name: Wait for RustFS
         run: |
-          echo "Waiting for RustFS to start..."
-          sleep 3
-          for i in {1..30}; do
-            # Try health endpoint
+          for i in {1..60}; do
             if curl -sf http://localhost:9000/health > /dev/null 2>&1; then
-              echo "RustFS is ready!"
+              echo "RustFS is ready"
               exit 0
             fi
-            echo "Waiting for RustFS... ($i/30)"
-            sleep 2
+            sleep 1
           done
           echo "RustFS failed to start"
           docker logs rustfs
           exit 1
 
-      - name: Run integration tests
+      - name: Run full integration suite
         run: cargo test --package rustfs-cli --test integration --features integration -- --test-threads=1
-        env:
-          TEST_S3_ENDPOINT: http://localhost:9000
-          TEST_S3_ACCESS_KEY: accesskey
-          TEST_S3_SECRET_KEY: secretkey
 
       - name: Show RustFS logs on failure
         if: failure()
-        run: docker logs rustfs 2>&1 | tail -100
+        run: docker logs rustfs 2>&1 | tail -200
 
   golden:
     name: Golden Tests
     runs-on: ubuntu-latest
-    # Don't block PR merges
-    continue-on-error: true
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
@@ -79,5 +138,4 @@ jobs:
       - name: Run golden tests
         run: cargo test --package rustfs-cli --test golden --features golden
         env:
-          # Golden tests use isolated config dir, no real S3 needed for alias tests
           RC_CONFIG_DIR: ${{ runner.temp }}/rc-test-config

crates/cli/src/commands/mv.rs

@@ -3,7 +3,7 @@
 //! Moves objects between locations (copy + delete).
 
 use clap::Args;
-use rc_core::{AliasManager, ObjectStore as _, ParsedPath, RemotePath, parse_path};
+use rc_core::{AliasManager, ListOptions, ObjectStore as _, ParsedPath, RemotePath, parse_path};
 use rc_s3::S3Client;
 use serde::Serialize;
 use std::path::{Path, PathBuf};
@@ -273,39 +273,168 @@ async fn move_s3_to_s3(
         return ExitCode::Success;
     }
 
-    // Copy
-    match client.copy_object(src, dst).await {
-        Ok(info) => {
-            // Delete source
-            if let Err(e) = client.delete_object(src).await {
-                formatter.error(&format!("Copied but failed to delete source: {e}"));
-                return ExitCode::GeneralError;
-            }
+    // Recursive move for prefix/directory semantics.
+    if args.recursive {
+        let mut continuation_token: Option<String> = None;
+        let mut moved_count = 0usize;
+        let mut error_count = 0usize;
+        let src_prefix = src.key.clone();
+
+        loop {
+            let list_opts = ListOptions {
+                recursive: true,
+                continuation_token: continuation_token.clone(),
+                ..Default::default()
+            };
+
+            let list_result = match client.list_objects(src, list_opts).await {
+                Ok(result) => result,
+                Err(e) => {
+                    formatter.error(&format!("Failed to list source objects: {e}"));
+                    return ExitCode::NetworkError;
+                }
+            };
+
+            for item in &list_result.items {
+                if item.is_dir {
+                    continue;
+                }
+
+                let relative = if src_prefix.is_empty() {
+                    item.key.clone()
+                } else if let Some(rest) = item.key.strip_prefix(&src_prefix) {
+                    rest.trim_start_matches('/').to_string()
+                } else {
+                    item.key.clone()
+                };
 
-            if formatter.is_json() {
-                let output = MvOutput {
-                    status: "success",
-                    source: src_display,
-                    target: dst_display,
-                    size_bytes: info.size_bytes,
+                let target_key = if dst.key.is_empty() {
+                    relative.clone()
+                } else if dst.key.ends_with('/') {
+                    format!("{}{}", dst.key, relative)
+                } else {
+                    format!("{}/{}", dst.key, relative)
                 };
-                formatter.json(&output);
-            } else {
-                formatter.println(&format!(
-                    "{src_display} -> {dst_display} ({})",
-                    info.size_human.unwrap_or_default()
-                ));
+
+                let src_obj = RemotePath::new(&src.alias, &src.bucket, &item.key);
+                let dst_obj = RemotePath::new(&dst.alias, &dst.bucket, &target_key);
+                let src_obj_display = src_obj.to_string();
+                let dst_obj_display = dst_obj.to_string();
+
+                match client.copy_object(&src_obj, &dst_obj).await {
+                    Ok(_) => match client.delete_object(&src_obj).await {
+                        Ok(()) => {
+                            moved_count += 1;
+                            if !formatter.is_json() {
+                                formatter
+                                    .println(&format!("{src_obj_display} -> {dst_obj_display}"));
+                            }
+                        }
+                        Err(e) => {
+                            error_count += 1;
+                            formatter.error(&format!(
+                                "Copied but failed to delete source '{src_obj_display}': {e}"
+                            ));
+                            if !args.continue_on_error {
+                                return ExitCode::GeneralError;
+                            }
+                        }
+                    },
+                    Err(e) => {
+                        error_count += 1;
+                        formatter.error(&format!(
+                            "Failed to move '{src_obj_display}' -> '{dst_obj_display}': {e}"
+                        ));
+                        if !args.continue_on_error {
+                            return ExitCode::NetworkError;
+                        }
+                    }
+                }
+            }
+
+            if !list_result.truncated {
+                break;
+            }
+            continuation_token = match list_result.continuation_token.clone() {
+                Some(token) => Some(token),
+                None => {
+                    formatter.error(
+                        "Backend indicated truncated results but did not provide a continuation token; stopping to avoid an infinite loop.",
+                    );
+                    return ExitCode::GeneralError;
+                }
+            };
+        }
+
+        if formatter.is_json() {
+            #[derive(Serialize)]
+            struct MvRecursiveOutput {
+                status: &'static str,
+                source: String,
+                target: String,
+                moved: usize,
+                errors: usize,
             }
+
+            formatter.json(&MvRecursiveOutput {
+                status: if error_count == 0 {
+                    "success"
+                } else {
+                    "partial"
+                },
+                source: src_display,
+                target: dst_display,
+                moved: moved_count,
+                errors: error_count,
+            });
+        } else if error_count == 0 {
+            formatter.println(&format!("Moved {moved_count} object(s)."));
+        } else {
+            formatter.println(&format!(
+                "Moved {moved_count} object(s), {error_count} failed."
+            ));
+        }
+
+        if error_count == 0 {
             ExitCode::Success
+        } else {
+            ExitCode::GeneralError
         }
-        Err(e) => {
-            let err_str = e.to_string();
-            if err_str.contains("NotFound") || err_str.contains("NoSuchKey") {
-                formatter.error(&format!("Source not found: {src_display}"));
-                ExitCode::NotFound
-            } else {
-                formatter.error(&format!("Failed to move: {e}"));
-                ExitCode::NetworkError
+    } else {
+        // Copy
+        match client.copy_object(src, dst).await {
+            Ok(info) => {
+                // Delete source
+                if let Err(e) = client.delete_object(src).await {
+                    formatter.error(&format!("Copied but failed to delete source: {e}"));
+                    return ExitCode::GeneralError;
+                }
+
+                if formatter.is_json() {
+                    let output = MvOutput {
+                        status: "success",
+                        source: src_display,
+                        target: dst_display,
+                        size_bytes: info.size_bytes,
+                    };
+                    formatter.json(&output);
+                } else {
+                    formatter.println(&format!(
+                        "{src_display} -> {dst_display} ({})",
+                        info.size_human.unwrap_or_default()
+                    ));
+                }
+                ExitCode::Success
+            }
+            Err(e) => {
+                let err_str = e.to_string();
+                if err_str.contains("NotFound") || err_str.contains("NoSuchKey") {
+                    formatter.error(&format!("Source not found: {src_display}"));
+                    ExitCode::NotFound
+                } else {
+                    formatter.error(&format!("Failed to move: {e}"));
+                    ExitCode::NetworkError
+                }
             }
         }
     }