Add support for overriding github owned storage multipart upload chunk size via environment variable

RELEASENOTES.md

@@ -1 +1 @@
-
+- Added support for configurable multipart upload chunk size for GitHub-owned storage uploads via `GITHUB_OWNED_STORAGE_MULTIPART_BYTES` environment variable (minimum 5 MiB, default 100 MiB) to improve upload reliability in environments with proxies or slow connections

src/Octoshift/Factories/GithubApiFactory.cs

@@ -30,7 +30,7 @@ GithubApi ISourceGithubApiFactory.Create(string apiUrl, string sourcePersonalAcc
         apiUrl ??= DEFAULT_API_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -39,7 +39,7 @@ GithubApi ISourceGithubApiFactory.CreateClientNoSsl(string apiUrl, string source
         apiUrl ??= DEFAULT_API_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("NoSSL"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -48,7 +48,7 @@ GithubApi ITargetGithubApiFactory.Create(string apiUrl, string targetPersonalAcc
         apiUrl ??= DEFAULT_API_URL;
         targetPersonalAccessToken ??= _environmentVariableProvider.TargetGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, targetPersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 }

src/Octoshift/Services/ArchiveUploader.cs

@@ -12,18 +12,24 @@ namespace OctoshiftCLI.Services;
 
 public class ArchiveUploader
 {
+    private const int MIN_MULTIPART_BYTES = 5 * 1024 * 1024; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
+
     private readonly GithubClient _client;
     private readonly OctoLogger _log;
+    private readonly EnvironmentVariableProvider _environmentVariableProvider;
     internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
     private readonly RetryPolicy _retryPolicy;
 
     private const string BASE_URL = "https://uploads.github.com";
 
-    public ArchiveUploader(GithubClient client, OctoLogger log, RetryPolicy retryPolicy)
+    public ArchiveUploader(GithubClient client, OctoLogger log, RetryPolicy retryPolicy, EnvironmentVariableProvider environmentVariableProvider)
     {
         _client = client;
         _log = log;
         _retryPolicy = retryPolicy;
+        _environmentVariableProvider = environmentVariableProvider;
+
+        SetStreamSizeLimitFromEnvironment();
     }
     public virtual async Task<string> Upload(Stream archiveContent, string archiveName, string orgDatabaseId)
     {
@@ -160,4 +166,22 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
         }
         throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload.");
     }
+
+    private void SetStreamSizeLimitFromEnvironment()
+    {
+        var envValue = _environmentVariableProvider.GithubOwnedStorageMultipartBytes();
+        if (!int.TryParse(envValue, out var limit) || limit <= 0)
+        {
+            return;
+        }
+
+        if (limit < MIN_MULTIPART_BYTES)
+        {
+            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_BYTES is set to {limit} bytes, but the minimum value is {MIN_MULTIPART_BYTES} bytes. Using default value of {_streamSizeLimit} bytes.");
+            return;
+        }
+
+        _streamSizeLimit = limit;
+        _log.LogInformation($"Stream size limit set to {_streamSizeLimit} bytes.");
+    }
 }

src/Octoshift/Services/EnvironmentVariableProvider.cs

@@ -18,6 +18,7 @@ public class EnvironmentVariableProvider
     private const string SMB_PASSWORD = "SMB_PASSWORD";
     private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK";
     private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK";
+    private const string GITHUB_OWNED_STORAGE_MULTIPART_BYTES = "GITHUB_OWNED_STORAGE_MULTIPART_BYTES";
 
     private readonly OctoLogger _logger;
 
@@ -65,6 +66,9 @@ public virtual string SkipStatusCheck(bool throwIfNotFound = false) =>
     public virtual string SkipVersionCheck(bool throwIfNotFound = false) =>
         GetValue(GEI_SKIP_VERSION_CHECK, throwIfNotFound);
 
+    public virtual string GithubOwnedStorageMultipartBytes(bool throwIfNotFound = false) =>
+        GetValue(GITHUB_OWNED_STORAGE_MULTIPART_BYTES, throwIfNotFound);
+
     private string GetValue(string name, bool throwIfNotFound)
     {
         var value = Environment.GetEnvironmentVariable(name);

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -60,7 +60,8 @@ public BbsToGithub(ITestOutputHelper output)
         _targetGithubHttpClient = new HttpClient();
         _targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger), new DateTimeProvider(), targetGithubToken);
         var retryPolicy = new RetryPolicy(_logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, _logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(_logger);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, _logger, retryPolicy, environmentVariableProvider);
         _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger), _archiveUploader);
 
         _blobServiceClient = new BlobServiceClient(_azureStorageConnectionString);

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -48,10 +48,11 @@ public GhesToGithub(ITestOutputHelper output)
 
         _versionClient = new HttpClient();
         var retryPolicy = new RetryPolicy(logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(logger);
 
         _sourceGithubHttpClient = new HttpClient();
         _sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), sourceGithubToken);
+        _archiveUploader = new ArchiveUploader(_sourceGithubClient, logger, retryPolicy, environmentVariableProvider);
         _sourceGithubApi = new GithubApi(_sourceGithubClient, GHES_API_URL, new RetryPolicy(logger), _archiveUploader);
 
         _targetGithubHttpClient = new HttpClient();