From e0fdb976a2cf3fe085f410efadaddd38e81b1fc8 Mon Sep 17 00:00:00 2001
From: Bassem Dghaidi <568794+Link-@users.noreply.github.com>
Date: Mon, 2 Dec 2024 07:01:28 -0800
Subject: [PATCH] Tune upload options
---
dist/restore-only/index.js | 16 ++++++++++++++--
dist/restore/index.js | 16 ++++++++++++++--
dist/save-only/index.js | 16 ++++++++++++++--
dist/save/index.js | 16 ++++++++++++++--
4 files changed, 56 insertions(+), 8 deletions(-)
diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js
index a59278e..7813066 100644
--- a/dist/restore-only/index.js
+++ b/dist/restore-only/index.js
@@ -6231,7 +6231,7 @@ function saveCacheV1(paths, key, options, enableCrossOsArchive = false) {
function saveCacheV2(paths, key, options, enableCrossOsArchive = false) {
return __awaiter(this, void 0, void 0, function* () {
// Override UploadOptions to force the use of Azure
- options = Object.assign(Object.assign({}, options), { uploadChunkSize: 64 * 1024 * 1024, uploadConcurrency: 8, useAzureSdk: true });
+ options = Object.assign(Object.assign({}, options), { useAzureSdk: true });
const compressionMethod = yield utils.getCompressionMethod();
const twirpClient = cacheTwirpClient.internalCacheTwirpClient();
let cacheId = -1;
@@ -9909,10 +9909,11 @@ const core = __importStar(__nccwpck_require__(4850));
* @param copy the original upload options
*/
function getUploadOptions(copy) {
+ // Defaults if not overriden
const result = {
useAzureSdk: false,
uploadConcurrency: 4,
- uploadChunkSize: 32 * 1024 * 1024
+ uploadChunkSize: 64 * 1024 * 1024
};
if (copy) {
if (typeof copy.useAzureSdk === 'boolean') {
@@ -9925,6 +9926,17 @@ function getUploadOptions(copy) {
result.uploadChunkSize = copy.uploadChunkSize;
}
}
+ /**
+ * Add env var overrides
+ */
+ // Cap the uploadConcurrency at 32
+ result.uploadConcurrency = !isNaN(Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ : result.uploadConcurrency;
+ // Cap the uploadChunkSize at 128MiB
+ result.uploadChunkSize = !isNaN(Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']))
+ ? Math.min(128 * 1024 * 1024, Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024)
+ : result.uploadChunkSize;
core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
core.debug(`Upload concurrency: ${result.uploadConcurrency}`);
core.debug(`Upload chunk size: ${result.uploadChunkSize}`);
diff --git a/dist/restore/index.js b/dist/restore/index.js
index c3ac999..335a3a3 100644
--- a/dist/restore/index.js
+++ b/dist/restore/index.js
@@ -6231,7 +6231,7 @@ function saveCacheV1(paths, key, options, enableCrossOsArchive = false) {
function saveCacheV2(paths, key, options, enableCrossOsArchive = false) {
return __awaiter(this, void 0, void 0, function* () {
// Override UploadOptions to force the use of Azure
- options = Object.assign(Object.assign({}, options), { uploadChunkSize: 64 * 1024 * 1024, uploadConcurrency: 8, useAzureSdk: true });
+ options = Object.assign(Object.assign({}, options), { useAzureSdk: true });
const compressionMethod = yield utils.getCompressionMethod();
const twirpClient = cacheTwirpClient.internalCacheTwirpClient();
let cacheId = -1;
@@ -9909,10 +9909,11 @@ const core = __importStar(__nccwpck_require__(4850));
* @param copy the original upload options
*/
function getUploadOptions(copy) {
+ // Defaults if not overriden
const result = {
useAzureSdk: false,
uploadConcurrency: 4,
- uploadChunkSize: 32 * 1024 * 1024
+ uploadChunkSize: 64 * 1024 * 1024
};
if (copy) {
if (typeof copy.useAzureSdk === 'boolean') {
@@ -9925,6 +9926,17 @@ function getUploadOptions(copy) {
result.uploadChunkSize = copy.uploadChunkSize;
}
}
+ /**
+ * Add env var overrides
+ */
+ // Cap the uploadConcurrency at 32
+ result.uploadConcurrency = !isNaN(Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ : result.uploadConcurrency;
+ // Cap the uploadChunkSize at 128MiB
+ result.uploadChunkSize = !isNaN(Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']))
+ ? Math.min(128 * 1024 * 1024, Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024)
+ : result.uploadChunkSize;
core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
core.debug(`Upload concurrency: ${result.uploadConcurrency}`);
core.debug(`Upload chunk size: ${result.uploadChunkSize}`);
diff --git a/dist/save-only/index.js b/dist/save-only/index.js
index 615b307..3b3aab5 100644
--- a/dist/save-only/index.js
+++ b/dist/save-only/index.js
@@ -6231,7 +6231,7 @@ function saveCacheV1(paths, key, options, enableCrossOsArchive = false) {
function saveCacheV2(paths, key, options, enableCrossOsArchive = false) {
return __awaiter(this, void 0, void 0, function* () {
// Override UploadOptions to force the use of Azure
- options = Object.assign(Object.assign({}, options), { uploadChunkSize: 64 * 1024 * 1024, uploadConcurrency: 8, useAzureSdk: true });
+ options = Object.assign(Object.assign({}, options), { useAzureSdk: true });
const compressionMethod = yield utils.getCompressionMethod();
const twirpClient = cacheTwirpClient.internalCacheTwirpClient();
let cacheId = -1;
@@ -9909,10 +9909,11 @@ const core = __importStar(__nccwpck_require__(4850));
* @param copy the original upload options
*/
function getUploadOptions(copy) {
+ // Defaults if not overriden
const result = {
useAzureSdk: false,
uploadConcurrency: 4,
- uploadChunkSize: 32 * 1024 * 1024
+ uploadChunkSize: 64 * 1024 * 1024
};
if (copy) {
if (typeof copy.useAzureSdk === 'boolean') {
@@ -9925,6 +9926,17 @@ function getUploadOptions(copy) {
result.uploadChunkSize = copy.uploadChunkSize;
}
}
+ /**
+ * Add env var overrides
+ */
+ // Cap the uploadConcurrency at 32
+ result.uploadConcurrency = !isNaN(Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ : result.uploadConcurrency;
+ // Cap the uploadChunkSize at 128MiB
+ result.uploadChunkSize = !isNaN(Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']))
+ ? Math.min(128 * 1024 * 1024, Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024)
+ : result.uploadChunkSize;
core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
core.debug(`Upload concurrency: ${result.uploadConcurrency}`);
core.debug(`Upload chunk size: ${result.uploadChunkSize}`);
diff --git a/dist/save/index.js b/dist/save/index.js
index 688f230..3018d9d 100644
--- a/dist/save/index.js
+++ b/dist/save/index.js
@@ -6231,7 +6231,7 @@ function saveCacheV1(paths, key, options, enableCrossOsArchive = false) {
function saveCacheV2(paths, key, options, enableCrossOsArchive = false) {
return __awaiter(this, void 0, void 0, function* () {
// Override UploadOptions to force the use of Azure
- options = Object.assign(Object.assign({}, options), { uploadChunkSize: 64 * 1024 * 1024, uploadConcurrency: 8, useAzureSdk: true });
+ options = Object.assign(Object.assign({}, options), { useAzureSdk: true });
const compressionMethod = yield utils.getCompressionMethod();
const twirpClient = cacheTwirpClient.internalCacheTwirpClient();
let cacheId = -1;
@@ -9909,10 +9909,11 @@ const core = __importStar(__nccwpck_require__(4850));
* @param copy the original upload options
*/
function getUploadOptions(copy) {
+ // Defaults if not overriden
const result = {
useAzureSdk: false,
uploadConcurrency: 4,
- uploadChunkSize: 32 * 1024 * 1024
+ uploadChunkSize: 64 * 1024 * 1024
};
if (copy) {
if (typeof copy.useAzureSdk === 'boolean') {
@@ -9925,6 +9926,17 @@ function getUploadOptions(copy) {
result.uploadChunkSize = copy.uploadChunkSize;
}
}
+ /**
+ * Add env var overrides
+ */
+ // Cap the uploadConcurrency at 32
+ result.uploadConcurrency = !isNaN(Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+ : result.uploadConcurrency;
+ // Cap the uploadChunkSize at 128MiB
+ result.uploadChunkSize = !isNaN(Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']))
+ ? Math.min(128 * 1024 * 1024, Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024)
+ : result.uploadChunkSize;
core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
core.debug(`Upload concurrency: ${result.uploadConcurrency}`);
core.debug(`Upload chunk size: ${result.uploadChunkSize}`);