Skip to content

Commit b5f7f78

Browse files
peterhuenepeterhuene
authored
feat: extract HTTP client. (#7)
* feat: extract HTTP client. This commit extracts out the internal HTTP client so that users of the library can configure their own HTTP client or share the client between all copy operations. Also changes `Alphanumeric` to use upper and lower case characters by default; users can use the alternate format to use only lower case. * chore: update CHANGELOG.
1 parent f762206 commit b5f7f78

File tree

10 files changed

+179
-115
lines changed

10 files changed

+179
-115
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
#### Added
1111

12+
* Added `HttpClient` for allowing reuse of an HTTP client between copy
13+
operations ([#7](https://github.com/stjude-rust-labs/cloud-copy/pull/7)).
1214
* Added support for linking files to the cache ([#6](https://github.com/stjude-rust-labs/cloud-copy/pull/6)).
1315
* Added statistics to the CLI upon successful copy ([#5](https://github.com/stjude-rust-labs/cloud-copy/pull/5)).
1416
* Added `--cache-dir` CLI option for download caching ([#5](https://github.com/stjude-rust-labs/cloud-copy/pull/5)).

‎src/backend/azure.rs‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ use reqwest::Body;
1212
use reqwest::Response;
1313
use reqwest::StatusCode;
1414
use reqwest::header;
15-
use reqwest_middleware::ClientWithMiddleware;
1615
use serde::Deserialize;
1716
use serde::Serialize;
1817
use tokio::sync::broadcast;
@@ -22,6 +21,7 @@ use url::Url;
2221
use crate::BLOCK_SIZE_THRESHOLD;
2322
use crate::Config;
2423
use crate::Error;
24+
use crate::HttpClient;
2525
use crate::ONE_MEBIBYTE;
2626
use crate::Result;
2727
use crate::TransferEvent;
@@ -30,7 +30,6 @@ use crate::UrlExt;
3030
use crate::backend::StorageBackend;
3131
use crate::backend::Upload;
3232
use crate::generator::Alphanumeric;
33-
use crate::new_http_client;
3433
use crate::streams::ByteStream;
3534
use crate::streams::TransferStream;
3635

@@ -179,7 +178,7 @@ impl ResponseExt for Response {
179178
/// Represents an upload of a blob to Azure Blob Storage.
180179
pub struct AzureBlobUpload {
181180
/// The HTTP client to use for the upload.
182-
client: ClientWithMiddleware,
181+
client: HttpClient,
183182
/// The blob URL.
184183
url: Url,
185184
/// The Azure block id.
@@ -191,7 +190,7 @@ pub struct AzureBlobUpload {
191190
impl AzureBlobUpload {
192191
/// Constructs a new blob upload.
193192
fn new(
194-
client: ClientWithMiddleware,
193+
client: HttpClient,
195194
url: Url,
196195
block_id: Arc<String>,
197196
events: Option<broadcast::Sender<TransferEvent>>,
@@ -304,24 +303,22 @@ pub struct AzureBlobStorageBackend {
304303
/// The config to use for transferring files.
305304
config: Config,
306305
/// The HTTP client to use for transferring files.
307-
client: ClientWithMiddleware,
308-
/// The HTTP cache used by the client.
309-
///
310-
/// This is `None` if caching is not enabled.
311-
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
306+
client: HttpClient,
312307
/// The channel for sending transfer events.
313308
events: Option<broadcast::Sender<TransferEvent>>,
314309
}
315310

316311
impl AzureBlobStorageBackend {
317312
/// Constructs a new Azure Blob Storage backend with the given configuration
318313
/// and events channel.
319-
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
320-
let (client, cache) = new_http_client(&config);
314+
pub fn new(
315+
config: Config,
316+
client: HttpClient,
317+
events: Option<broadcast::Sender<TransferEvent>>,
318+
) -> Self {
321319
Self {
322320
config,
323321
client,
324-
cache,
325322
events,
326323
}
327324
}
@@ -335,7 +332,7 @@ impl StorageBackend for AzureBlobStorageBackend {
335332
}
336333

337334
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
338-
self.cache.as_deref()
335+
self.client.cache()
339336
}
340337

341338
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {

‎src/backend/generic.rs‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,26 @@
22
//!
33
//! The generic storage backend can only be used for downloading files.
44
5-
use std::sync::Arc;
6-
75
use bytes::Bytes;
86
use chrono::Utc;
97
use http_cache_stream_reqwest::Cache;
108
use http_cache_stream_reqwest::storage::DefaultCacheStorage;
119
use reqwest::Response;
1210
use reqwest::StatusCode;
1311
use reqwest::header;
14-
use reqwest_middleware::ClientWithMiddleware;
1512
use tokio::sync::broadcast;
1613
use tracing::debug;
1714
use url::Url;
1815

1916
use crate::Config;
2017
use crate::Error;
18+
use crate::HttpClient;
2119
use crate::Result;
2220
use crate::TransferEvent;
2321
use crate::USER_AGENT;
2422
use crate::UrlExt;
2523
use crate::backend::StorageBackend;
2624
use crate::backend::Upload;
27-
use crate::new_http_client;
2825

2926
/// Helper trait for converting responses into `Error`.
3027
trait IntoError {
@@ -72,24 +69,22 @@ pub struct GenericStorageBackend {
7269
/// The configuration to use for transferring files.
7370
config: Config,
7471
/// The HTTP client to use for transferring files.
75-
client: ClientWithMiddleware,
76-
/// The HTTP cache used by the client.
77-
///
78-
/// This is `None` if caching is not enabled.
79-
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
72+
client: HttpClient,
8073
/// The channel for sending transfer events.
8174
events: Option<broadcast::Sender<TransferEvent>>,
8275
}
8376

8477
impl GenericStorageBackend {
8578
/// Constructs a new generic storage backend with the given configuration
8679
/// and events channel.
87-
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
88-
let (client, cache) = new_http_client(&config);
80+
pub fn new(
81+
config: Config,
82+
client: HttpClient,
83+
events: Option<broadcast::Sender<TransferEvent>>,
84+
) -> Self {
8985
Self {
9086
config,
9187
client,
92-
cache,
9388
events,
9489
}
9590
}
@@ -103,7 +98,7 @@ impl StorageBackend for GenericStorageBackend {
10398
}
10499

105100
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
106-
self.cache.as_deref()
101+
self.client.cache()
107102
}
108103

109104
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {

‎src/backend/google.rs‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ use reqwest::Response;
1313
use reqwest::StatusCode;
1414
use reqwest::header;
1515
use reqwest::header::HeaderValue;
16-
use reqwest_middleware::ClientWithMiddleware;
1716
use secrecy::ExposeSecret;
1817
use serde::Deserialize;
1918
use serde::Serialize;
@@ -25,6 +24,7 @@ use crate::BLOCK_SIZE_THRESHOLD;
2524
use crate::Config;
2625
use crate::Error;
2726
use crate::GoogleAuthConfig;
27+
use crate::HttpClient;
2828
use crate::ONE_MEBIBYTE;
2929
use crate::Result;
3030
use crate::TransferEvent;
@@ -37,7 +37,6 @@ use crate::backend::auth::SignatureProvider;
3737
use crate::backend::auth::sha256_hex_string;
3838
use crate::backend::s3::InitiateMultipartUploadResult;
3939
use crate::backend::s3::ListBucketResult;
40-
use crate::new_http_client;
4140
use crate::streams::ByteStream;
4241
use crate::streams::TransferStream;
4342

@@ -262,7 +261,7 @@ pub struct GoogleUpload {
262261
/// The configuration to use for the upload.
263262
config: Arc<Config>,
264263
/// The HTTP client to use for uploading.
265-
client: ClientWithMiddleware,
264+
client: HttpClient,
266265
/// The URL of the object being uploaded.
267266
url: Url,
268267
/// The identifier of this upload.
@@ -397,23 +396,21 @@ pub struct GoogleStorageBackend {
397396
/// The config to use for transferring files.
398397
config: Arc<Config>,
399398
/// The HTTP client to use for transferring files.
400-
client: ClientWithMiddleware,
401-
/// The HTTP cache used by the client.
402-
///
403-
/// This is `None` if caching is not enabled.
404-
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
399+
client: HttpClient,
405400
/// The channel for sending transfer events.
406401
events: Option<broadcast::Sender<TransferEvent>>,
407402
}
408403

409404
impl GoogleStorageBackend {
410405
/// Constructs a new Google Cloud Storage backend.
411-
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
412-
let (client, cache) = new_http_client(&config);
406+
pub fn new(
407+
config: Config,
408+
client: HttpClient,
409+
events: Option<broadcast::Sender<TransferEvent>>,
410+
) -> Self {
413411
Self {
414412
config: Arc::new(config),
415413
client,
416-
cache,
417414
events,
418415
}
419416
}
@@ -427,7 +424,7 @@ impl StorageBackend for GoogleStorageBackend {
427424
}
428425

429426
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
430-
self.cache.as_deref()
427+
self.client.cache()
431428
}
432429

433430
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {

‎src/backend/s3.rs‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ use reqwest::Response;
1313
use reqwest::StatusCode;
1414
use reqwest::header;
1515
use reqwest::header::HeaderValue;
16-
use reqwest_middleware::ClientWithMiddleware;
1716
use secrecy::ExposeSecret;
1817
use serde::Deserialize;
1918
use serde::Serialize;
@@ -24,6 +23,7 @@ use url::Url;
2423
use crate::BLOCK_SIZE_THRESHOLD;
2524
use crate::Config;
2625
use crate::Error;
26+
use crate::HttpClient;
2727
use crate::ONE_MEBIBYTE;
2828
use crate::Result;
2929
use crate::S3AuthConfig;
@@ -35,7 +35,6 @@ use crate::backend::Upload;
3535
use crate::backend::auth::RequestSigner;
3636
use crate::backend::auth::SignatureProvider;
3737
use crate::backend::auth::sha256_hex_string;
38-
use crate::new_http_client;
3938
use crate::streams::ByteStream;
4039
use crate::streams::TransferStream;
4140

@@ -334,7 +333,7 @@ pub struct S3Upload {
334333
/// The configuration to use for the upload.
335334
config: Arc<Config>,
336335
/// The HTTP client to use for uploading.
337-
client: ClientWithMiddleware,
336+
client: HttpClient,
338337
/// The URL of the object being uploaded.
339338
url: Url,
340339
/// The identifier of this upload.
@@ -464,23 +463,21 @@ pub struct S3StorageBackend {
464463
/// The config to use for transferring files.
465464
config: Arc<Config>,
466465
/// The HTTP client to use for transferring files.
467-
client: ClientWithMiddleware,
468-
/// The HTTP cache used by the client.
469-
///
470-
/// This is `None` if caching is not enabled.
471-
cache: Option<Arc<Cache<DefaultCacheStorage>>>,
466+
client: HttpClient,
472467
/// The channel for sending transfer events.
473468
events: Option<broadcast::Sender<TransferEvent>>,
474469
}
475470

476471
impl S3StorageBackend {
477472
/// Constructs a new S3 storage backend.
478-
pub fn new(config: Config, events: Option<broadcast::Sender<TransferEvent>>) -> Self {
479-
let (client, cache) = new_http_client(&config);
473+
pub fn new(
474+
config: Config,
475+
client: HttpClient,
476+
events: Option<broadcast::Sender<TransferEvent>>,
477+
) -> Self {
480478
Self {
481479
config: Arc::new(config),
482480
client,
483-
cache,
484481
events,
485482
}
486483
}
@@ -494,7 +491,7 @@ impl StorageBackend for S3StorageBackend {
494491
}
495492

496493
fn cache(&self) -> Option<&Cache<DefaultCacheStorage>> {
497-
self.cache.as_deref()
494+
self.client.cache()
498495
}
499496

500497
fn events(&self) -> &Option<broadcast::Sender<TransferEvent>> {

‎src/config.rs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
//! Implementation of cloud configuration.
22
33
use std::num::NonZero;
4-
use std::path::PathBuf;
54
use std::thread::available_parallelism;
65
use std::time::Duration;
76

@@ -83,11 +82,6 @@ pub struct GoogleConfig {
8382
/// Configuration used in a cloud copy operation.
8483
#[derive(Debug, Clone, Default, Deserialize)]
8584
pub struct Config {
86-
/// The cache directory to use for downloads.
87-
///
88-
/// If `None`, downloads will not use a cache.
89-
#[serde(default)]
90-
pub cache_dir: Option<PathBuf>,
9185
/// If `link_to_cache` is `true`, then a downloaded file that is already
9286
/// present (and fresh) in the cache will be hard linked at the requested
9387
/// destination instead of copied.

‎src/generator.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ use rand::Rng as _;
77
/// An alphanumeric string generator.
88
///
99
/// Every display of the generator will create a new random alphanumeric string.
10+
///
11+
/// By default, the displayed string will use all alphanumeric characters.
12+
///
13+
/// Use the alternate format (i.e. `{:#}`) to use only lowercase alphanumeric
14+
/// characters.
1015
#[derive(Clone, Copy)]
1116
pub struct Alphanumeric {
1217
/// The length of the alphanumeric string.
@@ -29,10 +34,11 @@ impl Default for Alphanumeric {
2934

3035
impl fmt::Display for Alphanumeric {
3136
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
37+
let lowercase = f.alternate();
3238
for c in rand::rng()
3339
.sample_iter(&rand::distr::Alphanumeric)
3440
.take(self.length)
35-
.map(|c| c.to_ascii_lowercase())
41+
.map(|c| if lowercase { c.to_ascii_lowercase() } else { c })
3642
{
3743
write!(f, "{c}", c = c as char)?;
3844
}

0 commit comments

Comments
 (0)