Skip to content

Commit f762206

Browse files
peterhueneclaymcleod
peterhuene
and
claymcleod
authored
feat: implement link to cached files. (#6)
* feat: implement link to cached files. This commit implements creating hard links to cached files instead of making copies. If a hard link cannot be made, a copy is performed instead. This commit also fixes the use case when `copy` is called to copy between local file paths (supported, although not particularly useful). The copy was being performed with a direct call to `fs::copy`, which did not emit any transfer events. As a result, the statistics printed at the end of the run was always 0 files copied, 0 bytes transferred. The fix was to wrap the source file stream with a transfer stream so that transfer events are emitted and the statistics are then updated. * chore: update CHANGELOG. * fix: make transfer identifiers monotonic across all transfers. * chore: refactor `cli` feature code. This commit moves the `cli` feature code into the `cli` module. Additionally, makes the `DisplayTimeDelta` utility type from `main.rs` as an extension method on `TimeDelta` so that other CLI implementations can use it. * Update src/cli.rs Co-authored-by: Clay McLeod <[email protected]> --------- Co-authored-by: Clay McLeod <[email protected]>
1 parent 2b6eb2e commit f762206

File tree

13 files changed

+578
-239
lines changed

13 files changed

+578
-239
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
#### Added
1111

12+
* Added support for linking files to the cache ([#6](https://github.com/stjude-rust-labs/cloud-copy/pull/6)).
1213
* Added statistics to the CLI upon successful copy ([#5](https://github.com/stjude-rust-labs/cloud-copy/pull/5)).
1314
* Added `--cache-dir` CLI option for download caching ([#5](https://github.com/stjude-rust-labs/cloud-copy/pull/5)).
1415
* Added testing with emulators Azurite and localstack ([#4](https://github.com/stjude-rust-labs/cloud-copy/pull/4)).

‎src/backend.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
//! Implementation of storage backends.
22
33
use bytes::Bytes;
4+
use http_cache_stream_reqwest::Cache;
5+
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
46
use reqwest::Response;
57
use tokio::sync::broadcast;
68
use url::Url;
@@ -42,6 +44,11 @@ pub trait StorageBackend {
4244
/// Gets the configuration used by the backend.
4345
fn config(&self) -> &Config;
4446

47+
/// Gets the HTTP cache used by the backend.
48+
///
49+
/// Returns `None` if caching is not enabled.
50+
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>>;
51+
4552
/// Gets the channel for sending transfer events.
4653
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>>;
4754

‎src/backend/azure.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ use base64::Engine;
66
use base64::prelude::BASE64_STANDARD;
77
use chrono::Utc;
88
use crc64fast_nvme::Digest;
9+
use http_cache_stream_reqwest::Cache;
10+
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
911
use reqwest::Body;
1012
use reqwest::Response;
1113
use reqwest::StatusCode;
@@ -303,6 +305,10 @@ pub struct AzureBlobStorageBackend {
303305
config: Config,
304306
/// The HTTP client to use for transferring files.
305307
client: ClientWithMiddleware,
308+
/// The HTTP cache used by the client.
309+
///
310+
/// This is `None` if caching is not enabled.
311+
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
306312
/// The channel for sending transfer events.
307313
events: Option<broadcast::Sender<TransferEvent>>,
308314
}
@@ -311,10 +317,11 @@ impl AzureBlobStorageBackend {
311317
/// Constructs a new Azure Blob Storage backend with the given configuration
312318
/// and events channel.
313319
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
314-
let client = new_http_client(&config);
320+
let (client, cache) = new_http_client(&config);
315321
Self {
316322
config,
317323
client,
324+
cache,
318325
events,
319326
}
320327
}
@@ -327,6 +334,10 @@ impl StorageBackend for AzureBlobStorageBackend {
327334
&self.config
328335
}
329336

337+
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
338+
self.cache.as_deref()
339+
}
340+
330341
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {
331342
&self.events
332343
}

‎src/backend/generic.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,12 @@
22
//!
33
//! The generic storage backend can only be used for downloading files.
44
5+
use std::sync::Arc;
6+
57
use bytes::Bytes;
68
use chrono::Utc;
9+
use http_cache_stream_reqwest::Cache;
10+
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
711
use reqwest::Response;
812
use reqwest::StatusCode;
913
use reqwest::header;
@@ -69,6 +73,10 @@ pub struct GenericStorageBackend {
6973
config: Config,
7074
/// The HTTP client to use for transferring files.
7175
client: ClientWithMiddleware,
76+
/// The HTTP cache used by the client.
77+
///
78+
/// This is `None` if caching is not enabled.
79+
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
7280
/// The channel for sending transfer events.
7381
events: Option<broadcast::Sender<TransferEvent>>,
7482
}
@@ -77,10 +85,11 @@ impl GenericStorageBackend {
7785
/// Constructs a new generic storage backend with the given configuration
7886
/// and events channel.
7987
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
80-
let client = new_http_client(&config);
88+
let (client, cache) = new_http_client(&config);
8189
Self {
8290
config,
8391
client,
92+
cache,
8493
events,
8594
}
8695
}
@@ -93,6 +102,10 @@ impl StorageBackend for GenericStorageBackend {
93102
&self.config
94103
}
95104

105+
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
106+
self.cache.as_deref()
107+
}
108+
96109
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {
97110
&self.events
98111
}

‎src/backend/google.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ use std::sync::Arc;
55
use bytes::Bytes;
66
use chrono::DateTime;
77
use chrono::Utc;
8+
use http_cache_stream_reqwest::Cache;
9+
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
810
use reqwest::Body;
911
use reqwest::Request;
1012
use reqwest::Response;
@@ -396,17 +398,22 @@ pub struct GoogleStorageBackend {
396398
config: Arc<Config>,
397399
/// The HTTP client to use for transferring files.
398400
client: ClientWithMiddleware,
401+
/// The HTTP cache used by the client.
402+
///
403+
/// This is `None` if caching is not enabled.
404+
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
399405
/// The channel for sending transfer events.
400406
events: Option<broadcast::Sender<TransferEvent>>,
401407
}
402408

403409
impl GoogleStorageBackend {
404410
/// Constructs a new Google Cloud Storage backend.
405411
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
406-
let client = new_http_client(&config);
412+
let (client, cache) = new_http_client(&config);
407413
Self {
408414
config: Arc::new(config),
409415
client,
416+
cache,
410417
events,
411418
}
412419
}
@@ -419,6 +426,10 @@ impl StorageBackend for GoogleStorageBackend {
419426
&self.config
420427
}
421428

429+
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
430+
self.cache.as_deref()
431+
}
432+
422433
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {
423434
&self.events
424435
}

‎src/backend/s3.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ use std::sync::Arc;
55
use bytes::Bytes;
66
use chrono::DateTime;
77
use chrono::Utc;
8+
use http_cache_stream_reqwest::Cache;
9+
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
810
use reqwest::Body;
911
use reqwest::Request;
1012
use reqwest::Response;
@@ -463,17 +465,22 @@ pub struct S3StorageBackend {
463465
config: Arc<Config>,
464466
/// The HTTP client to use for transferring files.
465467
client: ClientWithMiddleware,
468+
/// The HTTP cache used by the client.
469+
///
470+
/// This is `None` if caching is not enabled.
471+
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
466472
/// The channel for sending transfer events.
467473
events: Option<broadcast::Sender<TransferEvent>>,
468474
}
469475

470476
impl S3StorageBackend {
471477
/// Constructs a new S3 storage backend.
472478
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
473-
let client = new_http_client(&config);
479+
let (client, cache) = new_http_client(&config);
474480
Self {
475481
config: Arc::new(config),
476482
client,
483+
cache,
477484
events,
478485
}
479486
}
@@ -486,6 +493,10 @@ impl StorageBackend for S3StorageBackend {
486493
&self.config
487494
}
488495

496+
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
497+
self.cache.as_deref()
498+
}
499+
489500
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {
490501
&self.events
491502
}

0 commit comments

Comments
 (0)