Merge pull request #1479 from github/brianaj/external-pr-1403

External pr 1403: Support configurable GHOS multipart upload chunk size

Author: brianaj

README.md

@@ -95,6 +95,24 @@ When the CLI is launched, it logs if a newer version of the CLI is available. Yo
 
 When the CLI is launched, it logs a warning if there are any ongoing [GitHub incidents](https://www.githubstatus.com/) that might affect your use of the CLI. You can skip this check by setting the `GEI_SKIP_STATUS_CHECK` environment variable to `true`. 
 
+### Configuring multipart upload chunk size
+
+Set the `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable to change the archive upload part size. Provide the value in mebibytes (MiB); For example:
+
+```powershell
+# Windows PowerShell
+$env:GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "10"
+```
+
+```bash
+# macOS/Linux
+export GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES=10
+```
+
+This sets the chunk size to 10 MiB (10,485,760 bytes). The minimum supported value is 5 MiB, and the default remains 100 MiB.
+
+This might be needed to improve upload reliability in environments with proxies or very slow connections.
+
 ## Contributions
 
 See [Contributing](CONTRIBUTING.md) for more info on how to get involved.

RELEASENOTES.md

@@ -1 +1 @@
-
+- Added support for configurable multipart upload chunk size for GitHub-owned storage uploads via `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable (minimum 5 MiB, default 100 MiB) to improve upload reliability in environments with proxies or slow connections

src/Octoshift/Factories/GithubApiFactory.cs

@@ -32,7 +32,7 @@ GithubApi ISourceGithubApiFactory.Create(string apiUrl, string uploadsUrl, strin
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -42,7 +42,7 @@ GithubApi ISourceGithubApiFactory.CreateClientNoSsl(string apiUrl, string upload
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("NoSSL"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -52,7 +52,7 @@ GithubApi ITargetGithubApiFactory.Create(string apiUrl, string uploadsUrl, strin
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         targetPersonalAccessToken ??= _environmentVariableProvider.TargetGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, targetPersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 }

src/Octoshift/Services/ArchiveUploader.cs

@@ -11,18 +11,26 @@ namespace OctoshiftCLI.Services;
 
 public class ArchiveUploader
 {
+    private const int BYTES_PER_MEBIBYTE = 1024 * 1024;
+    private const int MIN_MULTIPART_MEBIBYTES = 5; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
+    private const int DEFAULT_MULTIPART_MEBIBYTES = 100;
+
     private readonly GithubClient _client;
     private readonly string _uploadsUrl;
     private readonly OctoLogger _log;
-    internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
+    private readonly EnvironmentVariableProvider _environmentVariableProvider;
+    internal int _streamSizeLimit = DEFAULT_MULTIPART_MEBIBYTES * BYTES_PER_MEBIBYTE; // 100 MiB stored in bytes
     private readonly RetryPolicy _retryPolicy;
 
-    public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy)
+    public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy, EnvironmentVariableProvider environmentVariableProvider)
     {
         _client = client;
         _uploadsUrl = uploadsUrl;
         _log = log;
         _retryPolicy = retryPolicy;
+        _environmentVariableProvider = environmentVariableProvider;
+
+        SetStreamSizeLimitFromEnvironment();
     }
     public virtual async Task<string> Upload(Stream archiveContent, string archiveName, string orgDatabaseId)
     {
@@ -160,4 +168,23 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
         }
         throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload.");
     }
+
+    private void SetStreamSizeLimitFromEnvironment()
+    {
+        var envValue = _environmentVariableProvider.GithubOwnedStorageMultipartMebibytes();
+        if (!int.TryParse(envValue, out var limitInMebibytes) || limitInMebibytes <= 0)
+        {
+            return;
+        }
+
+        if (limitInMebibytes < MIN_MULTIPART_MEBIBYTES)
+        {
+            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES is set to {limitInMebibytes} MiB, but the minimum value is {MIN_MULTIPART_MEBIBYTES} MiB. Using default value of {DEFAULT_MULTIPART_MEBIBYTES} MiB.");
+            return;
+        }
+
+        var limitBytes = (int)((long)limitInMebibytes * BYTES_PER_MEBIBYTE);
+        _streamSizeLimit = limitBytes;
+        _log.LogInformation($"Multipart upload part size set to {limitInMebibytes} MiB.");
+    }
 }

src/Octoshift/Services/EnvironmentVariableProvider.cs

@@ -18,6 +18,7 @@ public class EnvironmentVariableProvider
     private const string SMB_PASSWORD = "SMB_PASSWORD";
     private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK";
     private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK";
+    private const string GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES";
 
     private readonly OctoLogger _logger;
 
@@ -65,6 +66,9 @@ public virtual string SkipStatusCheck(bool throwIfNotFound = false) =>
     public virtual string SkipVersionCheck(bool throwIfNotFound = false) =>
         GetValue(GEI_SKIP_VERSION_CHECK, throwIfNotFound);
 
+    public virtual string GithubOwnedStorageMultipartMebibytes(bool throwIfNotFound = false) =>
+        GetValue(GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES, throwIfNotFound);
+
     private string GetValue(string name, bool throwIfNotFound)
     {
         var value = Environment.GetEnvironmentVariable(name);

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -60,7 +60,8 @@ public BbsToGithub(ITestOutputHelper output)
         _targetGithubHttpClient = new HttpClient();
         _targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger), new DateTimeProvider(), targetGithubToken);
         var retryPolicy = new RetryPolicy(_logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, _logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(_logger);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, _logger, retryPolicy, environmentVariableProvider);
         _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger), _archiveUploader);
 
         _blobServiceClient = new BlobServiceClient(_azureStorageConnectionString);

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -49,10 +49,11 @@ public GhesToGithub(ITestOutputHelper output)
 
         _versionClient = new HttpClient();
         var retryPolicy = new RetryPolicy(logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(logger);
 
         _sourceGithubHttpClient = new HttpClient();
         _sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), sourceGithubToken);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy, environmentVariableProvider);
         _sourceGithubApi = new GithubApi(_sourceGithubClient, GHES_API_URL, new RetryPolicy(logger), _archiveUploader);
 
         _targetGithubHttpClient = new HttpClient();