perf: improve performance with part sizing

package.json

@@ -81,7 +81,7 @@
     "@aws-sdk/credential-providers": "^3.1000.0",
     "@smithy/shared-ini-file-loader": "^4.4.5",
     "@tigrisdata/iam": "^1.3.0",
-    "@tigrisdata/storage": "^2.15.1",
+    "@tigrisdata/storage": "^2.15.2",
     "axios": "^1.13.6",
     "commander": "^14.0.3",
     "enquirer": "^2.4.1",

src/auth/s3-client.ts

@@ -54,9 +54,17 @@ export type TigrisStorageConfig = {
   organizationId?: string;
   iamEndpoint?: string;
   authDomain?: string;
+  credentialProvider?: () => Promise<{
+    accessKeyId: string;
+    secretAccessKey: string;
+    sessionToken?: string;
+    expiration?: Date;
+  }>;
 };
 
-export async function getStorageConfig(): Promise<TigrisStorageConfig> {
+export async function getStorageConfig(options?: {
+  withCredentialProvider?: boolean;
+}): Promise<TigrisStorageConfig> {
   // 1. AWS profile (only if AWS_PROFILE is set)
   if (hasAwsProfile()) {
     const profile = process.env.AWS_PROFILE || 'default';
@@ -78,7 +86,6 @@ export async function getStorageConfig(): Promise<TigrisStorageConfig> {
 
   if (loginMethod === 'oauth') {
     const authClient = getAuthClient();
-    const accessToken = await authClient.getAccessToken();
     const selectedOrg = getSelectedOrganization();
 
     if (!selectedOrg) {
@@ -88,9 +95,20 @@ export async function getStorageConfig(): Promise<TigrisStorageConfig> {
     }
 
     return {
-      sessionToken: accessToken,
+      sessionToken: await authClient.getAccessToken(),
       accessKeyId: '',
       secretAccessKey: '',
+      // Only include credentialProvider for long-running operations (uploads)
+      // that need token refresh. Short-lived operations (ls, rm, head) use
+      // the static sessionToken above and benefit from S3Client caching.
+      ...(options?.withCredentialProvider && {
+        credentialProvider: async () => ({
+          accessKeyId: '',
+          secretAccessKey: '',
+          sessionToken: await authClient.getAccessToken(),
+          expiration: new Date(Date.now() + 10 * 60 * 1000),
+        }),
+      }),
       endpoint: tigrisConfig.endpoint,
       organizationId: selectedOrg,
       iamEndpoint: tigrisConfig.iamEndpoint,
@@ -132,7 +150,7 @@ export async function getStorageConfig(): Promise<TigrisStorageConfig> {
 
   // No valid auth method found — try auto-login in interactive terminals
   if (await triggerAutoLogin()) {
-    return getStorageConfig();
+    return getStorageConfig(options);
   }
   throw new Error(
     'Not authenticated. Please run "tigris login" or "tigris configure" first.'
@@ -164,7 +182,6 @@ export async function getS3Client(): Promise<S3Client> {
 
   if (loginMethod === 'oauth') {
     const authClient = getAuthClient();
-    const accessToken = await authClient.getAccessToken();
     const selectedOrg = getSelectedOrganization();
 
     if (!selectedOrg) {
@@ -173,14 +190,17 @@ export async function getS3Client(): Promise<S3Client> {
       );
     }
 
+    const credentialProvider = async () => ({
+      accessKeyId: '',
+      secretAccessKey: '',
+      sessionToken: await authClient.getAccessToken(),
+      expiration: new Date(Date.now() + 10 * 60 * 1000),
+    });
+
     const client = new S3Client({
       region: 'auto',
       endpoint: tigrisConfig.endpoint,
-      credentials: {
-        sessionToken: accessToken,
-        accessKeyId: '', // Required by SDK but not used with token auth
-        secretAccessKey: '', // Required by SDK but not used with token auth
-      },
+      credentials: credentialProvider,
     });
 
     // Add middleware to inject custom headers

src/lib/cp.ts

@@ -23,6 +23,7 @@ import { getStorageConfig } from '../auth/s3-client.js';
 import { formatSize } from '../utils/format.js';
 import { get, put, list, head } from '@tigrisdata/storage';
 import { executeWithConcurrency } from '../utils/concurrency.js';
+import { calculateUploadParams } from '../utils/upload.js';
 import type { ParsedPath } from '../types.js';
 
 type CopyDirection = 'local-to-remote' | 'remote-to-local' | 'remote-to-remote';
@@ -106,15 +107,13 @@ async function uploadFile(
     return { error: `File not found: ${localPath}` };
   }
 
-  const fileStream = createReadStream(localPath);
+  const fileStream = createReadStream(localPath, {
+    highWaterMark: 1024 * 1024,
+  });
   const body = Readable.toWeb(fileStream) as ReadableStream;
 
-  const useMultipart = fileSize !== undefined && fileSize > 16 * 1024 * 1024;
-
   const { error: putError } = await put(key, body, {
-    multipart: useMultipart,
-    partSize: useMultipart ? 16 * 1024 * 1024 : undefined,
-    queueSize: useMultipart ? 8 : undefined,
+    ...calculateUploadParams(fileSize),
     onUploadProgress: showProgress
       ? ({ loaded }) => {
           if (fileSize !== undefined && fileSize > 0) {
@@ -246,12 +245,8 @@ async function copyObject(
     return { error: getError.message };
   }
 
-  const useMultipart = fileSize !== undefined && fileSize > 16 * 1024 * 1024;
-
   const { error: putError } = await put(destKey, data, {
-    multipart: useMultipart,
-    partSize: useMultipart ? 16 * 1024 * 1024 : undefined,
-    queueSize: useMultipart ? 8 : undefined,
+    ...calculateUploadParams(fileSize),
     onUploadProgress: showProgress
       ? ({ loaded }) => {
           if (fileSize !== undefined && fileSize > 0) {
@@ -749,7 +744,7 @@ export default async function cp(options: Record<string, unknown>) {
 
   const recursive = !!getOption<boolean>(options, ['recursive', 'r']);
   const direction = detectDirection(src, dest);
-  const config = await getStorageConfig();
+  const config = await getStorageConfig({ withCredentialProvider: true });
 
   switch (direction) {
     case 'local-to-remote': {

src/lib/mv.ts

@@ -11,6 +11,7 @@ import { getOption } from '../utils/options.js';
 import { getStorageConfig } from '../auth/s3-client.js';
 import { formatSize } from '../utils/format.js';
 import { get, put, remove, list, head } from '@tigrisdata/storage';
+import { calculateUploadParams } from '../utils/upload.js';
 
 async function confirm(message: string): Promise<boolean> {
   const rl = readline.createInterface({
@@ -66,7 +67,7 @@ export default async function mv(options: Record<string, unknown>) {
     process.exit(1);
   }
 
-  const config = await getStorageConfig();
+  const config = await getStorageConfig({ withCredentialProvider: true });
 
   // Check if source is a single object or a prefix (folder/wildcard)
   const isWildcard = srcPath.path.includes('*');
@@ -338,17 +339,14 @@ async function moveObject(
     return {};
   }
 
-  // Get source object size for progress
-  let fileSize: number | undefined;
-  if (showProgress) {
-    const { data: headData } = await head(srcKey, {
-      config: {
-        ...config,
-        bucket: srcBucket,
-      },
-    });
-    fileSize = headData?.size;
-  }
+  // Get source object size for upload params and progress
+  const { data: headData } = await head(srcKey, {
+    config: {
+      ...config,
+      bucket: srcBucket,
+    },
+  });
+  const fileSize = headData?.size;
 
   // Get source object
   const { data, error: getError } = await get(srcKey, 'stream', {
@@ -362,12 +360,9 @@ async function moveObject(
     return { error: getError.message };
   }
 
-  // Use multipart for files larger than 100MB
-  const useMultipart = fileSize !== undefined && fileSize > 100 * 1024 * 1024;
-
   // Put to destination
   const { error: putError } = await put(destKey, data, {
-    multipart: useMultipart,
+    ...calculateUploadParams(fileSize),
     onUploadProgress: showProgress
       ? ({ loaded }) => {
           if (fileSize !== undefined && fileSize > 0) {

src/lib/objects/put.ts

@@ -10,6 +10,7 @@ import {
   printFailure,
   msg,
 } from '../../utils/messages.js';
+import { calculateUploadParams } from '../../utils/upload.js';
 
 const context = msg('objects', 'put');
 
@@ -58,25 +59,24 @@ export default async function putObject(options: Record<string, unknown>) {
       printFailure(context, `File not found: ${file}`);
       process.exit(1);
     }
-    const fileStream = createReadStream(file);
+    const fileStream = createReadStream(file, { highWaterMark: 1024 * 1024 });
     body = Readable.toWeb(fileStream) as ReadableStream;
   } else {
     // Read from stdin
     body = Readable.toWeb(process.stdin) as ReadableStream;
   }
 
-  const config = await getStorageConfig();
+  const config = await getStorageConfig({ withCredentialProvider: true });
 
-  // Use multipart upload for files larger than 16MB (or always for stdin)
-  const useMultipart =
-    !file || (fileSize !== undefined && fileSize > 16 * 1024 * 1024);
+  // For stdin (no file), always use multipart since we don't know the size
+  const uploadParams = file
+    ? calculateUploadParams(fileSize)
+    : { multipart: true, partSize: 5 * 1024 * 1024, queueSize: 10 };
 
   const { data, error } = await put(key, body, {
     access: access === 'public' ? 'public' : 'private',
     contentType,
-    multipart: useMultipart,
-    partSize: useMultipart ? 16 * 1024 * 1024 : undefined,
-    queueSize: useMultipart ? 8 : undefined,
+    ...uploadParams,
     onUploadProgress: ({ loaded, percentage }) => {
       if (fileSize !== undefined && fileSize > 0) {
         process.stdout.write(

src/utils/upload.ts

@@ -0,0 +1,28 @@
+const MIN_PART_SIZE = 5 * 1024 * 1024; // 5 MB (S3 minimum)
+const MAX_PARTS = 10_000; // S3 hard limit
+const DEFAULT_QUEUE_SIZE = 10; // match AWS CLI
+
+// Tiered part sizes to balance parallelism vs per-part overhead
+const ONE_GB = 1024 * 1024 * 1024;
+const TEN_GB = 10 * ONE_GB;
+
+function tieredPartSize(fileSize: number): number {
+  if (fileSize <= ONE_GB) return 5 * 1024 * 1024; // 5 MB — max parallelism
+  if (fileSize <= TEN_GB) return 16 * 1024 * 1024; // 16 MB — fewer parts, less overhead
+  return 32 * 1024 * 1024; // 32 MB — very large files
+}
+
+export function calculateUploadParams(fileSize?: number) {
+  if (!fileSize || fileSize <= MIN_PART_SIZE) {
+    return { multipart: false } as const;
+  }
+
+  let partSize = tieredPartSize(fileSize);
+
+  // Safety: ensure we don't exceed S3's 10,000-part limit
+  if (fileSize / partSize > MAX_PARTS) {
+    partSize = Math.ceil(fileSize / MAX_PARTS);
+  }
+
+  return { multipart: true, partSize, queueSize: DEFAULT_QUEUE_SIZE } as const;
+}