Skip to content

Commit 76047c7

Browse files
osama-rizkosama-rizk
committed
Add remove multiple API
1 parent cefa453 commit 76047c7

File tree

14 files changed

+1292
-0
lines changed

14 files changed

+1292
-0
lines changed

‎packages/storage/examples/removeMultiple-example.ts‎

Lines changed: 321 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,321 @@
1+
/* eslint-disable no-console */
2+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
import { removeMultiple } from '../src/providers/s3';
6+
7+
/**
8+
* Example usage of the removeMultiple API with cancellation support
9+
*/
10+
async function exampleUsage() {
11+
try {
12+
// Basic usage - delete multiple files with cancellation support
13+
const basicOperation = removeMultiple({
14+
keys: [
15+
{ key: 'documents/file1.pdf' },
16+
{ key: 'images/photo1.jpg' },
17+
{ key: 'videos/video1.mp4' },
18+
],
19+
});
20+
21+
// Cancel after 2 seconds if needed
22+
setTimeout(() => {
23+
basicOperation.cancel();
24+
console.log('Basic operation cancelled');
25+
}, 2000);
26+
27+
const basicResult = await basicOperation.result;
28+
console.log(
29+
`Basic deletion: ${basicResult.summary.successCount} succeeded, ${basicResult.summary.failureCount} failed, ${basicResult.summary.cancelledCount} cancelled`,
30+
);
31+
32+
// Advanced usage with all options and cancellation
33+
const advancedOperation = removeMultiple({
34+
keys: [
35+
{ key: 'temp/file1.txt' },
36+
{ key: 'temp/file2.txt', versionId: 'version123' },
37+
{ key: 'temp/file3.txt' },
38+
// ... up to 5000 files
39+
],
40+
options: {
41+
batchSize: 500, // Process 500 files per batch
42+
batchStrategy: 'parallel', // Process batches in parallel
43+
maxConcurrency: 3, // Max 3 concurrent batches
44+
errorHandling: 'continue', // Continue even if some batches fail
45+
delayBetweenBatchesMs: 100, // 100ms delay between sequential batches
46+
retry: {
47+
maxAttempts: 3, // Retry failed batches up to 3 times
48+
delayMs: 1000, // 1 second delay between retries
49+
},
50+
onProgress: progress => {
51+
console.log(
52+
`Progress: ${progress.processedCount}/${progress.totalCount} processed`,
53+
);
54+
console.log(
55+
`Batch ${progress.batchNumber}: ${progress.currentBatch.successful.length} succeeded, ${progress.currentBatch.failed.length} failed`,
56+
);
57+
},
58+
},
59+
});
60+
61+
// Cancel after 10 seconds
62+
setTimeout(() => {
63+
if (!advancedOperation.isCancelled()) {
64+
advancedOperation.cancel();
65+
console.log('Advanced operation cancelled');
66+
}
67+
}, 10000);
68+
69+
const advancedResult = await advancedOperation.result;
70+
71+
console.log('Advanced deletion results:');
72+
console.log(`Total requested: ${advancedResult.summary.totalRequested}`);
73+
console.log(`Successfully deleted: ${advancedResult.summary.successCount}`);
74+
console.log(`Failed to delete: ${advancedResult.summary.failureCount}`);
75+
console.log(`Cancelled: ${advancedResult.summary.cancelledCount}`);
76+
console.log(`Was cancelled: ${advancedResult.summary.wasCancelled}`);
77+
console.log(
78+
`Batches processed: ${advancedResult.summary.batchesProcessed}`,
79+
);
80+
console.log(`Batches failed: ${advancedResult.summary.batchesFailed}`);
81+
82+
// Handle failures
83+
if (advancedResult.failed.length > 0) {
84+
console.log('Failed deletions:');
85+
advancedResult.failed.forEach(failure => {
86+
console.log(
87+
`- ${failure.key}: ${failure.error.message} (batch ${failure.error.batchNumber}, ${failure.error.retryAttempts} attempts)`,
88+
);
89+
});
90+
}
91+
92+
// Handle cancelled keys
93+
if (advancedResult.cancelled && advancedResult.cancelled.length > 0) {
94+
console.log('Cancelled deletions:');
95+
advancedResult.cancelled.forEach(cancelled => {
96+
console.log(`- ${cancelled.key} (batch ${cancelled.batchNumber})`);
97+
});
98+
}
99+
100+
// Sequential processing with cancellation during delay
101+
const sequentialOperation = removeMultiple({
102+
keys: [
103+
{ key: 'logs/log1.txt' },
104+
{ key: 'logs/log2.txt' },
105+
{ key: 'logs/log3.txt' },
106+
],
107+
options: {
108+
batchStrategy: 'sequential',
109+
errorHandling: 'failEarly', // Stop on first batch failure
110+
delayBetweenBatchesMs: 2000, // 2 second delay between batches
111+
},
112+
});
113+
114+
// Cancel during delay period - should stop immediately
115+
setTimeout(() => {
116+
sequentialOperation.cancel();
117+
console.log('Sequential operation cancelled during delay');
118+
}, 3000);
119+
120+
const sequentialResult = await sequentialOperation.result;
121+
console.log(
122+
`Sequential deletion: ${sequentialResult.summary.successCount} files deleted, ${sequentialResult.summary.cancelledCount} cancelled`,
123+
);
124+
} catch (error) {
125+
console.error('Deletion failed:', error);
126+
127+
// Handle validation errors
128+
if (
129+
error instanceof Error &&
130+
error.message.includes('Dangerous key detected')
131+
) {
132+
console.error(
133+
'Attempted to delete root folder or use dangerous patterns',
134+
);
135+
}
136+
}
137+
}
138+
139+
// Cancellation examples
140+
async function cancellationExamples() {
141+
// Example 1: Cancel immediately
142+
const operation1 = removeMultiple({
143+
keys: [{ key: 'file1.txt' }, { key: 'file2.txt' }],
144+
});
145+
146+
// Cancel before any processing starts
147+
operation1.cancel();
148+
149+
const result1 = await operation1.result;
150+
console.log('Immediate cancellation:', result1.summary.wasCancelled); // true
151+
console.log('All keys cancelled:', result1.summary.cancelledCount === 2); // true
152+
153+
// Example 2: Cancel during processing
154+
const operation2 = removeMultiple({
155+
keys: Array.from({ length: 5000 }, (_, i) => ({ key: `file${i}.txt` })),
156+
options: {
157+
batchSize: 1000,
158+
batchStrategy: 'sequential',
159+
delayBetweenBatchesMs: 1000,
160+
onProgress: progress => {
161+
console.log(`Processed batch ${progress.batchNumber}`);
162+
},
163+
},
164+
});
165+
166+
// Cancel after 2.5 seconds (likely during delay between batches)
167+
setTimeout(() => {
168+
console.log('Cancelling during processing...');
169+
operation2.cancel();
170+
}, 2500);
171+
172+
const result2 = await operation2.result;
173+
console.log('Partial processing results:');
174+
console.log(
175+
`- Processed: ${result2.summary.successCount + result2.summary.failureCount}`,
176+
);
177+
console.log(`- Cancelled: ${result2.summary.cancelledCount}`);
178+
console.log(`- Was cancelled: ${result2.summary.wasCancelled}`);
179+
180+
// Example 3: Check cancellation status
181+
const operation3 = removeMultiple({
182+
keys: [{ key: 'test.txt' }],
183+
});
184+
185+
console.log('Initially cancelled?', operation3.isCancelled()); // false
186+
187+
operation3.cancel();
188+
console.log('After cancel() called?', operation3.isCancelled()); // true
189+
190+
// Idempotent - safe to call multiple times
191+
operation3.cancel();
192+
operation3.cancel();
193+
console.log('After multiple cancel() calls?', operation3.isCancelled()); // still true
194+
195+
const result3 = await operation3.result;
196+
// Result still resolves normally, just with cancelled status
197+
console.log(
198+
'Result resolved with cancellation:',
199+
result3.summary.wasCancelled,
200+
);
201+
}
202+
203+
// Error handling with cancellation
204+
async function errorHandlingWithCancellation() {
205+
// Example: Cancel during retry attempts
206+
const operation = removeMultiple({
207+
keys: [{ key: 'file1.txt' }, { key: 'file2.txt' }],
208+
options: {
209+
retry: {
210+
maxAttempts: 5,
211+
delayMs: 2000, // 2 second delay between retries
212+
},
213+
errorHandling: 'continue',
214+
},
215+
});
216+
217+
// Cancel during retry attempts
218+
setTimeout(() => {
219+
console.log('Cancelling during retry attempts...');
220+
operation.cancel();
221+
}, 3000);
222+
223+
const result = await operation.result;
224+
225+
// Check if any failures were due to cancellation
226+
const cancelledFailures = result.failed.filter(
227+
f => f.error.code === 'CANCELLED',
228+
);
229+
console.log(`Failures due to cancellation: ${cancelledFailures.length}`);
230+
231+
// Result always resolves, never rejects, even when cancelled
232+
console.log(
233+
'Operation completed with cancellation:',
234+
result.summary.wasCancelled,
235+
);
236+
}
237+
238+
// UI Integration example
239+
function uiIntegrationExample() {
240+
let currentOperation: any = null;
241+
242+
// Start deletion
243+
function startDeletion(filesToDelete: { key: string }[]) {
244+
currentOperation = removeMultiple({
245+
keys: filesToDelete,
246+
options: {
247+
batchStrategy: 'parallel',
248+
maxConcurrency: 5,
249+
onProgress: progress => {
250+
// Update UI progress bar
251+
const percentage =
252+
(progress.processedCount / progress.totalCount) * 100;
253+
updateProgressBar(percentage);
254+
255+
// Update status text
256+
updateStatusText(
257+
`Processed ${progress.processedCount} of ${progress.totalCount} files`,
258+
);
259+
},
260+
},
261+
});
262+
263+
// Enable cancel button
264+
enableCancelButton();
265+
266+
// Handle completion
267+
currentOperation.result.then((result: any) => {
268+
if (result.summary.wasCancelled) {
269+
showMessage(
270+
`Operation cancelled. Processed ${result.summary.successCount} files.`,
271+
);
272+
} else {
273+
showMessage(
274+
`Deletion complete. ${result.summary.successCount} files deleted.`,
275+
);
276+
}
277+
278+
// Disable cancel button
279+
disableCancelButton();
280+
currentOperation = null;
281+
});
282+
}
283+
284+
// Cancel current operation
285+
function cancelDeletion() {
286+
if (currentOperation && !currentOperation.isCancelled()) {
287+
currentOperation.cancel();
288+
showMessage('Cancellation requested...');
289+
}
290+
}
291+
292+
// Mock UI functions
293+
function updateProgressBar(percentage: number) {
294+
console.log(`Progress: ${percentage.toFixed(1)}%`);
295+
}
296+
297+
function updateStatusText(text: string) {
298+
console.log(`Status: ${text}`);
299+
}
300+
301+
function showMessage(message: string) {
302+
console.log(`Message: ${message}`);
303+
}
304+
305+
function enableCancelButton() {
306+
console.log('Cancel button enabled');
307+
}
308+
309+
function disableCancelButton() {
310+
console.log('Cancel button disabled');
311+
}
312+
313+
return { startDeletion, cancelDeletion };
314+
}
315+
316+
export {
317+
exampleUsage,
318+
cancellationExamples,
319+
errorHandlingWithCancellation,
320+
uiIntegrationExample,
321+
};

‎packages/storage/src/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export {
1010
copy,
1111
getUrl,
1212
removeObjects,
13+
removeMultiple,
1314
} from './providers/s3';
1415

1516
export {
@@ -20,6 +21,8 @@ export {
2021
RemoveInput,
2122
RemoveWithPathInput,
2223
RemoveObjectsInput,
24+
RemoveMultipleInput,
25+
ProgressInfo,
2326
ListAllInput,
2427
ListAllWithPathInput,
2528
ListPaginateInput,
@@ -40,6 +43,8 @@ export {
4043
RemoveOutput,
4144
RemoveWithPathOutput,
4245
RemoveObjectsOutput,
46+
RemoveMultipleOutput,
47+
RemoveMultipleOperation,
4348
ListAllOutput,
4449
ListAllWithPathOutput,
4550
ListPaginateOutput,

‎packages/storage/src/internals/apis/removeMultiple.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
import { Amplify } from '@aws-amplify/core';
5+
6+
import { removeMultiple as removeMultipleInternal } from '../../providers/s3/apis/internal/removeMultiple';
7+
import { RemoveMultipleInput } from '../types/inputs';
8+
import { RemoveMultipleOperation } from '../types/outputs';
9+
10+
/**
11+
* @internal
12+
*/
13+
export const removeMultiple = (
14+
input: RemoveMultipleInput,
15+
): RemoveMultipleOperation =>
16+
removeMultipleInternal(Amplify, input) as RemoveMultipleOperation;

‎packages/storage/src/internals/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ export {
1919
ListPaginateInput,
2020
RemoveInput,
2121
RemoveObjectsInput,
22+
RemoveMultipleInput,
23+
ProgressInfo,
2224
UploadDataInput,
2325
DownloadDataInput,
2426
} from './types/inputs';
@@ -29,6 +31,8 @@ export {
2931
GetUrlOutput,
3032
RemoveOutput,
3133
RemoveObjectsOutput,
34+
RemoveMultipleOutput,
35+
RemoveMultipleOperation,
3236
UploadDataOutput,
3337
DownloadDataOutput,
3438
ListOutput,
@@ -42,6 +46,7 @@ export { getProperties } from './apis/getProperties';
4246
export { getUrl } from './apis/getUrl';
4347
export { remove } from './apis/remove';
4448
export { removeObjects } from './apis/removeObjects';
49+
export { removeMultiple } from './apis/removeMultiple';
4550
export { uploadData } from './apis/uploadData';
4651
export { downloadData } from './apis/downloadData';
4752
export { copy } from './apis/copy';

0 commit comments

Comments
 (0)