Gitea/cache
1
0
Fork 0
mirror of https://github.com/actions/cache.git synced 2025-11-04 15:48:39 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use tries on all API calls

Browse Source
This commit is contained in:
Dave Hadka 2020-05-10 10:36:06 -04:00
parent a8b61326cf
commit c8d75a8073
4 changed files with 214 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
__tests__/cacheHttpsClient.test.ts
View File

@ -59,20 +59,17 @@ function handleResponse(
async function testRetryExpectingResult(
responses: Array<TestResponse>,
expectedResult: string
expectedResult: string | null
): Promise<void> {
responses = responses.reverse(); // Reverse responses since we pop from end
const actualResult = await retry(
"test",
() => handleResponse(responses.pop()),
(response: TestResponse) => response.statusCode,
(response: TestResponse) => response.result,
(statusCode: number) => statusCode === 200,
(statusCode: number) => statusCode === 503
(response: TestResponse) => response.statusCode
);
expect(actualResult).toEqual(expectedResult);
expect(actualResult.result).toEqual(expectedResult);
}
async function testRetryExpectingError(
@ -84,10 +81,7 @@ async function testRetryExpectingError(
retry(
"test",
() => handleResponse(responses.pop()),
(response: TestResponse) => response.statusCode,
(response: TestResponse) => response.result,
(statusCode: number) => statusCode === 200,
(statusCode: number) => statusCode === 503
(response: TestResponse) => response.statusCode
)
).rejects.toBeInstanceOf(Error);
}
@ -165,3 +159,19 @@ test("retry works after error", async () => {
"Ok"
);
});
test("retry returns after client error", async () => {
await testRetryExpectingResult(
[
{
statusCode: 400,
result: null
},
{
statusCode: 200,
result: "Ok"
}
],
null
);
});

133
dist/restore/index.js vendored
View File

@ -2197,6 +2197,12 @@ function isSuccessStatusCode(statusCode) {
}
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode) {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode) {
if (!statusCode) {
return false;
@ -2246,7 +2252,7 @@ function getCacheVersion(compressionMethod) {
.digest("hex");
}
exports.getCacheVersion = getCacheVersion;
function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) {
function retry(name, method, getStatusCode, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
let response = undefined;
let statusCode = undefined;
@ -2257,8 +2263,8 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode,
try {
response = yield method();
statusCode = getStatusCode(response);
if (isSuccessStatusCode(statusCode)) {
return getReturnValue(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
errorMessage = `Cache service responded with ${statusCode}`;
@ -2280,13 +2286,13 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode,
exports.retry = retry;
function retryTypedResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts);
return yield retry(name, method, (response) => response.statusCode, maxAttempts);
});
}
exports.retryTypedResponse = retryTypedResponse;
function retryHttpClientResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts);
return yield retry(name, method, (response) => response.message.statusCode, maxAttempts);
});
}
exports.retryHttpClientResponse = retryHttpClientResponse;
@ -2322,7 +2328,7 @@ function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new http_client_1.HttpClient("actions/cache");
const downloadResponse = yield httpClient.get(archiveLocation);
const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation));
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => {
downloadResponse.message.destroy();
@ -2354,7 +2360,7 @@ function reserveCache(key, options) {
key,
version
};
const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest);
const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest));
return _d = (_c = (_b = response) === null || _b === void 0 ? void 0 : _b.result) === null || _c === void 0 ? void 0 : _c.cacheId, (_d !== null && _d !== void 0 ? _d : -1);
});
}
@ -2427,7 +2433,7 @@ function uploadFile(httpClient, cacheId, archivePath) {
function commitCache(httpClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const commitCacheRequest = { size: filesize };
return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest);
return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest));
});
}
function saveCache(cacheId, archivePath) {
@ -2469,7 +2475,9 @@ class BasicCredentialHandler {
this.password = password;
}
prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64');
options.headers['Authorization'] =
'Basic ' +
Buffer.from(this.username + ':' + this.password).toString('base64');
}
// This handler cannot handle 401
canHandleAuthentication(response) {
@ -2505,7 +2513,8 @@ class PersonalAccessTokenCredentialHandler {
// currently implements pre-authorization
// TODO: support preAuth = false where it hooks on 401
prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
options.headers['Authorization'] =
'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
}
// This handler cannot handle 401
canHandleAuthentication(response) {
@ -3231,8 +3240,8 @@ const core = __importStar(__webpack_require__(470));
const exec = __importStar(__webpack_require__(986));
const glob = __importStar(__webpack_require__(281));
const io = __importStar(__webpack_require__(1));
const os = __importStar(__webpack_require__(87));
const fs = __importStar(__webpack_require__(747));
const os = __importStar(__webpack_require__(87));
const path = __importStar(__webpack_require__(622));
const util = __importStar(__webpack_require__(669));
const uuidV4 = __importStar(__webpack_require__(826));
@ -3637,6 +3646,7 @@ var HttpCodes;
HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout";
HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict";
HttpCodes[HttpCodes["Gone"] = 410] = "Gone";
HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests";
HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError";
HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented";
HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway";
@ -3661,8 +3671,18 @@ function getProxyUrl(serverUrl) {
return proxyUrl ? proxyUrl.href : '';
}
exports.getProxyUrl = getProxyUrl;
const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect];
const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout];
const HttpRedirectCodes = [
HttpCodes.MovedPermanently,
HttpCodes.ResourceMoved,
HttpCodes.SeeOther,
HttpCodes.TemporaryRedirect,
HttpCodes.PermanentRedirect
];
const HttpResponseRetryCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD'];
const ExponentialBackoffCeiling = 10;
const ExponentialBackoffTimeSlice = 5;
@ -3676,12 +3696,6 @@ class HttpClientResponse {
this.message.on('data', (chunk) => {
output = Buffer.concat([output, chunk]);
});
this.message.on('aborted', () => {
reject("Request was aborted or closed prematurely");
});
this.message.on('timeout', (socket) => {
reject("Request timed out");
});
this.message.on('end', () => {
resolve(output.toString());
});
@ -3793,19 +3807,22 @@ class HttpClient {
*/
async request(verb, requestUrl, data, headers) {
if (this._disposed) {
throw new Error("Client has already been disposed.");
throw new Error('Client has already been disposed.');
}
let parsedUrl = url.parse(requestUrl);
let info = this._prepareRequest(verb, parsedUrl, headers);
// Only perform retries on reads since writes may not be idempotent.
let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1;
let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1
? this._maxRetries + 1
: 1;
let numTries = 0;
let response;
while (numTries < maxTries) {
response = await this.requestRaw(info, data);
// Check if it's an authentication challenge
if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) {
if (response &&
response.message &&
response.message.statusCode === HttpCodes.Unauthorized) {
let authenticationHandler;
for (let i = 0; i < this.handlers.length; i++) {
if (this.handlers[i].canHandleAuthentication(response)) {
@ -3823,21 +3840,32 @@ class HttpClient {
}
}
let redirectsRemaining = this._maxRedirects;
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1
&& this._allowRedirects
&& redirectsRemaining > 0) {
const redirectUrl = response.message.headers["location"];
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 &&
this._allowRedirects &&
redirectsRemaining > 0) {
const redirectUrl = response.message.headers['location'];
if (!redirectUrl) {
// if there's no location to redirect to, we won't
break;
}
let parsedRedirectUrl = url.parse(redirectUrl);
if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) {
throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.");
if (parsedUrl.protocol == 'https:' &&
parsedUrl.protocol != parsedRedirectUrl.protocol &&
!this._allowRedirectDowngrade) {
throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.');
}
// we need to finish reading the response before reassigning response
// which will leak the open socket.
await response.readBody();
// strip authorization header if redirected to a different hostname
if (parsedRedirectUrl.hostname !== parsedUrl.hostname) {
for (let header in headers) {
// header names are case insensitive
if (header.toLowerCase() === 'authorization') {
delete headers[header];
}
}
}
// let's make the request with the new redirectUrl
info = this._prepareRequest(verb, parsedRedirectUrl, headers);
response = await this.requestRaw(info, data);
@ -3888,8 +3916,8 @@ class HttpClient {
*/
requestRawWithCallback(info, data, onResult) {
let socket;
if (typeof (data) === 'string') {
info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8');
if (typeof data === 'string') {
info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8');
}
let callbackCalled = false;
let handleResult = (err, res) => {
@ -3902,7 +3930,7 @@ class HttpClient {
let res = new HttpClientResponse(msg);
handleResult(null, res);
});
req.on('socket', (sock) => {
req.on('socket', sock => {
socket = sock;
});
// If we ever get disconnected, we want the socket to timeout eventually
@ -3915,13 +3943,12 @@ class HttpClient {
req.on('error', function (err) {
// err has statusCode property
// res should have headers
console.log(`Caught error on request: ${err}`);
handleResult(err, null);
});
if (data && typeof (data) === 'string') {
if (data && typeof data === 'string') {
req.write(data, 'utf8');
}
if (data && typeof (data) !== 'string') {
if (data && typeof data !== 'string') {
data.on('close', function () {
req.end();
});
@ -3948,31 +3975,34 @@ class HttpClient {
const defaultPort = usingSsl ? 443 : 80;
info.options = {};
info.options.host = info.parsedUrl.hostname;
info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort;
info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.port = info.parsedUrl.port
? parseInt(info.parsedUrl.port)
: defaultPort;
info.options.path =
(info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.method = method;
info.options.headers = this._mergeHeaders(headers);
if (this.userAgent != null) {
info.options.headers["user-agent"] = this.userAgent;
info.options.headers['user-agent'] = this.userAgent;
}
info.options.agent = this._getAgent(info.parsedUrl);
// gives handlers an opportunity to participate
if (this.handlers) {
this.handlers.forEach((handler) => {
this.handlers.forEach(handler => {
handler.prepareRequest(info.options);
});
}
return info;
}
_mergeHeaders(headers) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {});
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
if (this.requestOptions && this.requestOptions.headers) {
return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers));
}
return lowercaseKeys(headers || {});
}
_getExistingOrDefaultHeader(additionalHeaders, header, _default) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {});
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
let clientHeader;
if (this.requestOptions && this.requestOptions.headers) {
clientHeader = lowercaseKeys(this.requestOptions.headers)[header];
@ -4010,7 +4040,7 @@ class HttpClient {
proxyAuth: proxyUrl.auth,
host: proxyUrl.hostname,
port: proxyUrl.port
},
}
};
let tunnelAgent;
const overHttps = proxyUrl.protocol === 'https:';
@ -4037,7 +4067,9 @@ class HttpClient {
// we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process
// http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options
// we have to cast it to any and change it directly
agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false });
agent.options = Object.assign(agent.options || {}, {
rejectUnauthorized: false
});
}
return agent;
}
@ -4098,7 +4130,7 @@ class HttpClient {
msg = contents;
}
else {
msg = "Failed request: (" + statusCode + ")";
msg = 'Failed request: (' + statusCode + ')';
}
let err = new Error(msg);
// attach statusCode and body obj (if available) to the error object
@ -5187,12 +5219,10 @@ function getProxyUrl(reqUrl) {
}
let proxyVar;
if (usingSsl) {
proxyVar = process.env["https_proxy"] ||
process.env["HTTPS_PROXY"];
proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY'];
}
else {
proxyVar = process.env["http_proxy"] ||
process.env["HTTP_PROXY"];
proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY'];
}
if (proxyVar) {
proxyUrl = url.parse(proxyVar);
@ -5204,7 +5234,7 @@ function checkBypass(reqUrl) {
if (!reqUrl.hostname) {
return false;
}
let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || '';
let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || '';
if (!noProxy) {
return false;
}
@ -5225,7 +5255,10 @@ function checkBypass(reqUrl) {
upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`);
}
// Compare request host against noproxy
for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) {
for (let upperNoProxyItem of noProxy
.split(',')
.map(x => x.trim().toUpperCase())
.filter(x => x)) {
if (upperReqHosts.some(x => x === upperNoProxyItem)) {
return true;
}

133
dist/save/index.js vendored
View File

@ -2197,6 +2197,12 @@ function isSuccessStatusCode(statusCode) {
}
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode) {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode) {
if (!statusCode) {
return false;
@ -2246,7 +2252,7 @@ function getCacheVersion(compressionMethod) {
.digest("hex");
}
exports.getCacheVersion = getCacheVersion;
function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) {
function retry(name, method, getStatusCode, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
let response = undefined;
let statusCode = undefined;
@ -2257,8 +2263,8 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode,
try {
response = yield method();
statusCode = getStatusCode(response);
if (isSuccessStatusCode(statusCode)) {
return getReturnValue(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
errorMessage = `Cache service responded with ${statusCode}`;
@ -2280,13 +2286,13 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode,
exports.retry = retry;
function retryTypedResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts);
return yield retry(name, method, (response) => response.statusCode, maxAttempts);
});
}
exports.retryTypedResponse = retryTypedResponse;
function retryHttpClientResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts);
return yield retry(name, method, (response) => response.message.statusCode, maxAttempts);
});
}
exports.retryHttpClientResponse = retryHttpClientResponse;
@ -2322,7 +2328,7 @@ function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new http_client_1.HttpClient("actions/cache");
const downloadResponse = yield httpClient.get(archiveLocation);
const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation));
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => {
downloadResponse.message.destroy();
@ -2354,7 +2360,7 @@ function reserveCache(key, options) {
key,
version
};
const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest);
const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest));
return _d = (_c = (_b = response) === null || _b === void 0 ? void 0 : _b.result) === null || _c === void 0 ? void 0 : _c.cacheId, (_d !== null && _d !== void 0 ? _d : -1);
});
}
@ -2427,7 +2433,7 @@ function uploadFile(httpClient, cacheId, archivePath) {
function commitCache(httpClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const commitCacheRequest = { size: filesize };
return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest);
return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest));
});
}
function saveCache(cacheId, archivePath) {
@ -2469,7 +2475,9 @@ class BasicCredentialHandler {
this.password = password;
}
prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64');
options.headers['Authorization'] =
'Basic ' +
Buffer.from(this.username + ':' + this.password).toString('base64');
}
// This handler cannot handle 401
canHandleAuthentication(response) {
@ -2505,7 +2513,8 @@ class PersonalAccessTokenCredentialHandler {
// currently implements pre-authorization
// TODO: support preAuth = false where it hooks on 401
prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
options.headers['Authorization'] =
'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
}
// This handler cannot handle 401
canHandleAuthentication(response) {
@ -3231,8 +3240,8 @@ const core = __importStar(__webpack_require__(470));
const exec = __importStar(__webpack_require__(986));
const glob = __importStar(__webpack_require__(281));
const io = __importStar(__webpack_require__(1));
const os = __importStar(__webpack_require__(87));
const fs = __importStar(__webpack_require__(747));
const os = __importStar(__webpack_require__(87));
const path = __importStar(__webpack_require__(622));
const util = __importStar(__webpack_require__(669));
const uuidV4 = __importStar(__webpack_require__(826));
@ -3637,6 +3646,7 @@ var HttpCodes;
HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout";
HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict";
HttpCodes[HttpCodes["Gone"] = 410] = "Gone";
HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests";
HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError";
HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented";
HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway";
@ -3661,8 +3671,18 @@ function getProxyUrl(serverUrl) {
return proxyUrl ? proxyUrl.href : '';
}
exports.getProxyUrl = getProxyUrl;
const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect];
const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout];
const HttpRedirectCodes = [
HttpCodes.MovedPermanently,
HttpCodes.ResourceMoved,
HttpCodes.SeeOther,
HttpCodes.TemporaryRedirect,
HttpCodes.PermanentRedirect
];
const HttpResponseRetryCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD'];
const ExponentialBackoffCeiling = 10;
const ExponentialBackoffTimeSlice = 5;
@ -3676,12 +3696,6 @@ class HttpClientResponse {
this.message.on('data', (chunk) => {
output = Buffer.concat([output, chunk]);
});
this.message.on('aborted', () => {
reject("Request was aborted or closed prematurely");
});
this.message.on('timeout', (socket) => {
reject("Request timed out");
});
this.message.on('end', () => {
resolve(output.toString());
});
@ -3793,19 +3807,22 @@ class HttpClient {
*/
async request(verb, requestUrl, data, headers) {
if (this._disposed) {
throw new Error("Client has already been disposed.");
throw new Error('Client has already been disposed.');
}
let parsedUrl = url.parse(requestUrl);
let info = this._prepareRequest(verb, parsedUrl, headers);
// Only perform retries on reads since writes may not be idempotent.
let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1;
let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1
? this._maxRetries + 1
: 1;
let numTries = 0;
let response;
while (numTries < maxTries) {
response = await this.requestRaw(info, data);
// Check if it's an authentication challenge
if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) {
if (response &&
response.message &&
response.message.statusCode === HttpCodes.Unauthorized) {
let authenticationHandler;
for (let i = 0; i < this.handlers.length; i++) {
if (this.handlers[i].canHandleAuthentication(response)) {
@ -3823,21 +3840,32 @@ class HttpClient {
}
}
let redirectsRemaining = this._maxRedirects;
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1
&& this._allowRedirects
&& redirectsRemaining > 0) {
const redirectUrl = response.message.headers["location"];
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 &&
this._allowRedirects &&
redirectsRemaining > 0) {
const redirectUrl = response.message.headers['location'];
if (!redirectUrl) {
// if there's no location to redirect to, we won't
break;
}
let parsedRedirectUrl = url.parse(redirectUrl);
if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) {
throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.");
if (parsedUrl.protocol == 'https:' &&
parsedUrl.protocol != parsedRedirectUrl.protocol &&
!this._allowRedirectDowngrade) {
throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.');
}
// we need to finish reading the response before reassigning response
// which will leak the open socket.
await response.readBody();
// strip authorization header if redirected to a different hostname
if (parsedRedirectUrl.hostname !== parsedUrl.hostname) {
for (let header in headers) {
// header names are case insensitive
if (header.toLowerCase() === 'authorization') {
delete headers[header];
}
}
}
// let's make the request with the new redirectUrl
info = this._prepareRequest(verb, parsedRedirectUrl, headers);
response = await this.requestRaw(info, data);
@ -3888,8 +3916,8 @@ class HttpClient {
*/
requestRawWithCallback(info, data, onResult) {
let socket;
if (typeof (data) === 'string') {
info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8');
if (typeof data === 'string') {
info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8');
}
let callbackCalled = false;
let handleResult = (err, res) => {
@ -3902,7 +3930,7 @@ class HttpClient {
let res = new HttpClientResponse(msg);
handleResult(null, res);
});
req.on('socket', (sock) => {
req.on('socket', sock => {
socket = sock;
});
// If we ever get disconnected, we want the socket to timeout eventually
@ -3915,13 +3943,12 @@ class HttpClient {
req.on('error', function (err) {
// err has statusCode property
// res should have headers
console.log(`Caught error on request: ${err}`);
handleResult(err, null);
});
if (data && typeof (data) === 'string') {
if (data && typeof data === 'string') {
req.write(data, 'utf8');
}
if (data && typeof (data) !== 'string') {
if (data && typeof data !== 'string') {
data.on('close', function () {
req.end();
});
@ -3948,31 +3975,34 @@ class HttpClient {
const defaultPort = usingSsl ? 443 : 80;
info.options = {};
info.options.host = info.parsedUrl.hostname;
info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort;
info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.port = info.parsedUrl.port
? parseInt(info.parsedUrl.port)
: defaultPort;
info.options.path =
(info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.method = method;
info.options.headers = this._mergeHeaders(headers);
if (this.userAgent != null) {
info.options.headers["user-agent"] = this.userAgent;
info.options.headers['user-agent'] = this.userAgent;
}
info.options.agent = this._getAgent(info.parsedUrl);
// gives handlers an opportunity to participate
if (this.handlers) {
this.handlers.forEach((handler) => {
this.handlers.forEach(handler => {
handler.prepareRequest(info.options);
});
}
return info;
}
_mergeHeaders(headers) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {});
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
if (this.requestOptions && this.requestOptions.headers) {
return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers));
}
return lowercaseKeys(headers || {});
}
_getExistingOrDefaultHeader(additionalHeaders, header, _default) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {});
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
let clientHeader;
if (this.requestOptions && this.requestOptions.headers) {
clientHeader = lowercaseKeys(this.requestOptions.headers)[header];
@ -4010,7 +4040,7 @@ class HttpClient {
proxyAuth: proxyUrl.auth,
host: proxyUrl.hostname,
port: proxyUrl.port
},
}
};
let tunnelAgent;
const overHttps = proxyUrl.protocol === 'https:';
@ -4037,7 +4067,9 @@ class HttpClient {
// we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process
// http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options
// we have to cast it to any and change it directly
agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false });
agent.options = Object.assign(agent.options || {}, {
rejectUnauthorized: false
});
}
return agent;
}
@ -4098,7 +4130,7 @@ class HttpClient {
msg = contents;
}
else {
msg = "Failed request: (" + statusCode + ")";
msg = 'Failed request: (' + statusCode + ')';
}
let err = new Error(msg);
// attach statusCode and body obj (if available) to the error object
@ -5164,12 +5196,10 @@ function getProxyUrl(reqUrl) {
}
let proxyVar;
if (usingSsl) {
proxyVar = process.env["https_proxy"] ||
process.env["HTTPS_PROXY"];
proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY'];
}
else {
proxyVar = process.env["http_proxy"] ||
process.env["HTTP_PROXY"];
proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY'];
}
if (proxyVar) {
proxyUrl = url.parse(proxyVar);
@ -5181,7 +5211,7 @@ function checkBypass(reqUrl) {
if (!reqUrl.hostname) {
return false;
}
let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || '';
let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || '';
if (!noProxy) {
return false;
}
@ -5202,7 +5232,10 @@ function checkBypass(reqUrl) {
upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`);
}
// Compare request host against noproxy
for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) {
for (let upperNoProxyItem of noProxy
.split(',')
.map(x => x.trim().toUpperCase())
.filter(x => x)) {
if (upperReqHosts.some(x => x === upperNoProxyItem)) {
return true;
}

42
src/cacheHttpClient.ts
View File

@ -30,6 +30,13 @@ function isSuccessStatusCode(statusCode?: number): boolean {
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode?: number): boolean {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode?: number): boolean {
if (!statusCode) {
return false;
@ -99,16 +106,13 @@ export function getCacheVersion(compressionMethod?: CompressionMethod): string {
.digest("hex");
}
export async function retry<R, T>(
export async function retry<T>(
name: string,
method: () => Promise<R>,
getStatusCode: (R) => number | undefined,
getReturnValue: (R) => T,
isSuccessStatusCode: (number) => boolean,
isRetryableStatusCode: (number) => boolean,
method: () => Promise<T>,
getStatusCode: (T) => number | undefined,
maxAttempts = 2
): Promise<T> {
let response: R | undefined = undefined;
let response: T | undefined = undefined;
let statusCode: number | undefined = undefined;
let isRetryable = false;
let errorMessage = "";
@ -119,8 +123,8 @@ export async function retry<R, T>(
response = await method();
statusCode = getStatusCode(response);
if (isSuccessStatusCode(statusCode)) {
return getReturnValue(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
@ -154,9 +158,6 @@ export async function retryTypedResponse<T>(
name,
method,
(response: ITypedResponse<T>) => response.statusCode,
(response: ITypedResponse<T>) => response,
isSuccessStatusCode,
isRetryableStatusCode,
maxAttempts
);
}
@ -170,9 +171,6 @@ export async function retryHttpClientResponse<T>(
name,
method,
(response: IHttpClientResponse) => response.message.statusCode,
(response: IHttpClientResponse) => response,
isSuccessStatusCode,
isRetryableStatusCode,
maxAttempts
);
}
@ -221,7 +219,10 @@ export async function downloadCache(
): Promise<void> {
const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient("actions/cache");
const downloadResponse = await httpClient.get(archiveLocation);
const downloadResponse = await retryHttpClientResponse(
"downloadCache",
() => httpClient.get(archiveLocation)
);
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
@ -263,10 +264,13 @@ export async function reserveCache(
key,
version
};
const response = await httpClient.postJson<ReserveCacheResponse>(
const response = await retryTypedResponse("reserveCache", () =>
httpClient.postJson<ReserveCacheResponse>(
getCacheApiUrl("caches"),
reserveCacheRequest
)
);
return response?.result?.cacheId ?? -1;
}
@ -381,9 +385,11 @@ async function commitCache(
filesize: number
): Promise<ITypedResponse<null>> {
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await httpClient.postJson<null>(
return await retryTypedResponse("commitCache", () =>
httpClient.postJson<null>(
getCacheApiUrl(`caches/${cacheId.toString()}`),
commitCacheRequest
)
);
}