From aced43a6506caadbe1a0f305388159af7c221cd7 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Fri, 8 May 2020 11:37:53 -0400 Subject: [PATCH 1/5] Fix uploadChunk and add generic retry method --- __tests__/cacheHttpsClient.test.ts | 130 ++++++++++++++++++++++++++++- dist/restore/index.js | 79 +++++++++++++----- dist/save/index.js | 79 +++++++++++++----- src/cacheHttpClient.ts | 124 ++++++++++++++++++++------- 4 files changed, 338 insertions(+), 74 deletions(-) diff --git a/__tests__/cacheHttpsClient.test.ts b/__tests__/cacheHttpsClient.test.ts index 362beb4..79e8210 100644 --- a/__tests__/cacheHttpsClient.test.ts +++ b/__tests__/cacheHttpsClient.test.ts @@ -1,4 +1,4 @@ -import { getCacheVersion } from "../src/cacheHttpClient"; +import { getCacheVersion, retry } from "../src/cacheHttpClient"; import { CompressionMethod, Inputs } from "../src/constants"; import * as testUtils from "../src/utils/testUtils"; @@ -37,3 +37,131 @@ test("getCacheVersion with gzip compression does not change vesion", async () => test("getCacheVersion with no input throws", async () => { expect(() => getCacheVersion()).toThrow(); }); + +interface TestResponse { + statusCode: number; + result: string | null; +} + +function handleResponse( + response: TestResponse | undefined +): Promise { + if (!response) { + fail("Retry method called too many times"); + } + + if (response.statusCode === 999) { + throw Error("Test Error"); + } else { + return Promise.resolve(response); + } +} + +async function testRetryExpectingResult( + responses: Array, + expectedResult: string +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + const actualResult = await retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode, + (response: TestResponse) => response.result, + (statusCode: number) => statusCode === 200, + (statusCode: number) => statusCode === 503 + ); + + expect(actualResult).toEqual(expectedResult); +} + +async function testRetryExpectingError( + responses: Array +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + expect( + retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode, + (response: TestResponse) => response.result, + (statusCode: number) => statusCode === 200, + (statusCode: number) => statusCode === 503 + ) + ).rejects.toBeInstanceOf(Error); +} + +test("retry works on successful response", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry works after retryable status code", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry fails after exhausting retries", async () => { + await testRetryExpectingError([ + { + statusCode: 503, + result: null + }, + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry fails after non-retryable status code", async () => { + await testRetryExpectingError([ + { + statusCode: 500, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry works after error", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 999, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); diff --git a/dist/restore/index.js b/dist/restore/index.js index 67707ad..f0b155d 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2246,19 +2246,60 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; +function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys, options) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const version = getCacheVersion((_a = options) === null || _a === void 0 ? void 0 : _a.compressionMethod); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}&version=${version}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = (_b = cacheResult) === null || _b === void 0 ? void 0 : _b.archiveLocation; if (!cacheDownloadUrl) { @@ -2326,7 +2367,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -2336,20 +2377,9 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, uploadChunkRequest); }); } function parseEnvNumber(key) { @@ -2379,13 +2409,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -3642,6 +3671,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3763,6 +3798,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -3874,6 +3910,7 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers + console.log(`Caught error on request: ${err}`); handleResult(err, null); }); if (data && typeof (data) === 'string') { diff --git a/dist/save/index.js b/dist/save/index.js index 3d7a6df..6dd837c 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2246,19 +2246,60 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; +function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys, options) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const version = getCacheVersion((_a = options) === null || _a === void 0 ? void 0 : _a.compressionMethod); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}&version=${version}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = (_b = cacheResult) === null || _b === void 0 ? void 0 : _b.archiveLocation; if (!cacheDownloadUrl) { @@ -2326,7 +2367,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -2336,20 +2377,9 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, uploadChunkRequest); }); } function parseEnvNumber(key) { @@ -2379,13 +2409,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -3642,6 +3671,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3763,6 +3798,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -3874,6 +3910,7 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers + console.log(`Caught error on request: ${err}`); handleResult(err, null); }); if (data && typeof (data) === 'string') { diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index c000b7f..7f06b6b 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -99,6 +99,84 @@ export function getCacheVersion(compressionMethod?: CompressionMethod): string { .digest("hex"); } +export async function retry( + name: string, + method: () => Promise, + getStatusCode: (R) => number | undefined, + getReturnValue: (R) => T, + isSuccessStatusCode: (number) => boolean, + isRetryableStatusCode: (number) => boolean, + maxAttempts = 2 +): Promise { + let response: R | undefined = undefined; + let statusCode: number | undefined = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + + while (attempt <= maxAttempts) { + try { + response = await method(); + statusCode = getStatusCode(response); + + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } catch (error) { + isRetryable = true; + errorMessage = error.message; + } + + core.debug( + `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}` + ); + + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + + attempt++; + } + + throw Error(`${name} failed: ${errorMessage}`); +} + +export async function retryTypedResponse( + name: string, + method: () => Promise>, + maxAttempts = 2 +): Promise> { + return await retry( + name, + method, + (response: ITypedResponse) => response.statusCode, + (response: ITypedResponse) => response, + isSuccessStatusCode, + isRetryableStatusCode, + maxAttempts + ); +} + +export async function retryHttpClientResponse( + name: string, + method: () => Promise, + maxAttempts = 2 +): Promise { + return await retry( + name, + method, + (response: IHttpClientResponse) => response.message.statusCode, + (response: IHttpClientResponse) => response, + isSuccessStatusCode, + isRetryableStatusCode, + maxAttempts + ); +} + export async function getCacheEntry( keys: string[], options?: CacheOptions @@ -109,15 +187,13 @@ export async function getCacheEntry( keys.join(",") )}&version=${version}`; - const response = await httpClient.getJson( - getCacheApiUrl(resource) + const response = await retryTypedResponse("getCacheEntry", () => + httpClient.getJson(getCacheApiUrl(resource)) ); + if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = cacheResult?.archiveLocation; @@ -206,7 +282,7 @@ function getContentRange(start: number, end: number): string { async function uploadChunk( httpClient: HttpClient, resourceUrl: string, - data: NodeJS.ReadableStream, + openStream: () => NodeJS.ReadableStream, start: number, end: number ): Promise { @@ -227,28 +303,14 @@ async function uploadChunk( return await httpClient.sendStream( "PATCH", resourceUrl, - data, + openStream(), additionalHeaders ); }; - const response = await uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug( - `Received ${response.message.statusCode}, retrying chunk at offset ${start}.` - ); - const retryResponse = await uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - - throw new Error( - `Cache service responded with ${response.message.statusCode} during chunk upload.` + await retryHttpClientResponse( + `uploadChunk (start: ${start}, end: ${end})`, + uploadChunkRequest ); } @@ -290,17 +352,17 @@ async function uploadFile( const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }); await uploadChunk( httpClient, resourceUrl, - chunk, + () => + fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end ); From 6efe05572db81ff16d35a7417f8e3835354690ca Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Fri, 8 May 2020 12:05:32 -0400 Subject: [PATCH 2/5] Test disabling concurrency --- dist/restore/index.js | 30 ++++++++++++++++-------------- dist/save/index.js | 30 ++++++++++++++++-------------- src/cacheHttpClient.ts | 8 ++++---- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index f0b155d..cf22bff 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2403,20 +2403,22 @@ function uploadFile(httpClient, cacheId, archivePath) { core.debug("Awaiting all uploads"); let offset = 0; try { - yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { - while (offset < fileSize) { - const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); - const start = offset; - const end = offset + chunkSize - 1; - offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }), start, end); - } - }))); + // await Promise.all( + // parallelUploads.map(async () => { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end); + } + // }) + // ); } finally { fs.closeSync(fd); diff --git a/dist/save/index.js b/dist/save/index.js index 6dd837c..4daea2a 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2403,20 +2403,22 @@ function uploadFile(httpClient, cacheId, archivePath) { core.debug("Awaiting all uploads"); let offset = 0; try { - yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { - while (offset < fileSize) { - const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); - const start = offset; - const end = offset + chunkSize - 1; - offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }), start, end); - } - }))); + // await Promise.all( + // parallelUploads.map(async () => { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end); + } + // }) + // ); } finally { fs.closeSync(fd); diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 7f06b6b..afa8e11 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -342,8 +342,8 @@ async function uploadFile( let offset = 0; try { - await Promise.all( - parallelUploads.map(async () => { +// await Promise.all( +// parallelUploads.map(async () => { while (offset < fileSize) { const chunkSize = Math.min( fileSize - offset, @@ -367,8 +367,8 @@ async function uploadFile( end ); } - }) - ); +// }) +// ); } finally { fs.closeSync(fd); } From 25b1a139de71dcf5d84935ec190231991decef56 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Fri, 8 May 2020 15:59:00 -0400 Subject: [PATCH 3/5] Revert "Test disabling concurrency" This reverts commit 6efe05572db81ff16d35a7417f8e3835354690ca. --- dist/restore/index.js | 30 ++++++++++++++---------------- dist/save/index.js | 30 ++++++++++++++---------------- src/cacheHttpClient.ts | 8 ++++---- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index cf22bff..f0b155d 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2403,22 +2403,20 @@ function uploadFile(httpClient, cacheId, archivePath) { core.debug("Awaiting all uploads"); let offset = 0; try { - // await Promise.all( - // parallelUploads.map(async () => { - while (offset < fileSize) { - const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); - const start = offset; - const end = offset + chunkSize - 1; - offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }), start, end); - } - // }) - // ); + yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end); + } + }))); } finally { fs.closeSync(fd); diff --git a/dist/save/index.js b/dist/save/index.js index 4daea2a..6dd837c 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2403,22 +2403,20 @@ function uploadFile(httpClient, cacheId, archivePath) { core.debug("Awaiting all uploads"); let offset = 0; try { - // await Promise.all( - // parallelUploads.map(async () => { - while (offset < fileSize) { - const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); - const start = offset; - const end = offset + chunkSize - 1; - offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }), start, end); - } - // }) - // ); + yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end); + } + }))); } finally { fs.closeSync(fd); diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index afa8e11..7f06b6b 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -342,8 +342,8 @@ async function uploadFile( let offset = 0; try { -// await Promise.all( -// parallelUploads.map(async () => { + await Promise.all( + parallelUploads.map(async () => { while (offset < fileSize) { const chunkSize = Math.min( fileSize - offset, @@ -367,8 +367,8 @@ async function uploadFile( end ); } -// }) -// ); + }) + ); } finally { fs.closeSync(fd); } From a8b61326cfd5a37b83a6c774ba84d329fe8e5767 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Fri, 8 May 2020 16:09:03 -0400 Subject: [PATCH 4/5] Disable zstd on Windows due to issue #301 --- dist/restore/index.js | 5 +++++ dist/save/index.js | 5 +++++ src/utils/actionUtils.ts | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/dist/restore/index.js b/dist/restore/index.js index f0b155d..d063a11 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -3231,6 +3231,7 @@ const core = __importStar(__webpack_require__(470)); const exec = __importStar(__webpack_require__(986)); const glob = __importStar(__webpack_require__(281)); const io = __importStar(__webpack_require__(1)); +const os = __importStar(__webpack_require__(87)); const fs = __importStar(__webpack_require__(747)); const path = __importStar(__webpack_require__(622)); const util = __importStar(__webpack_require__(669)); @@ -3372,6 +3373,10 @@ function getVersion(app) { } function getCompressionMethod() { return __awaiter(this, void 0, void 0, function* () { + // Disabling zstd on Windows due to https://github.com/actions/cache/issues/301 + if (os.platform() === "win32") { + return constants_1.CompressionMethod.Gzip; + } const versionOutput = yield getVersion("zstd"); return versionOutput.toLowerCase().includes("zstd command line interface") ? constants_1.CompressionMethod.Zstd diff --git a/dist/save/index.js b/dist/save/index.js index 6dd837c..7dba8da 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -3231,6 +3231,7 @@ const core = __importStar(__webpack_require__(470)); const exec = __importStar(__webpack_require__(986)); const glob = __importStar(__webpack_require__(281)); const io = __importStar(__webpack_require__(1)); +const os = __importStar(__webpack_require__(87)); const fs = __importStar(__webpack_require__(747)); const path = __importStar(__webpack_require__(622)); const util = __importStar(__webpack_require__(669)); @@ -3372,6 +3373,10 @@ function getVersion(app) { } function getCompressionMethod() { return __awaiter(this, void 0, void 0, function* () { + // Disabling zstd on Windows due to https://github.com/actions/cache/issues/301 + if (os.platform() === "win32") { + return constants_1.CompressionMethod.Gzip; + } const versionOutput = yield getVersion("zstd"); return versionOutput.toLowerCase().includes("zstd command line interface") ? constants_1.CompressionMethod.Zstd diff --git a/src/utils/actionUtils.ts b/src/utils/actionUtils.ts index 12c1c36..dd5c8a4 100644 --- a/src/utils/actionUtils.ts +++ b/src/utils/actionUtils.ts @@ -3,6 +3,7 @@ import * as exec from "@actions/exec"; import * as glob from "@actions/glob"; import * as io from "@actions/io"; import * as fs from "fs"; +import * as os from "os"; import * as path from "path"; import * as util from "util"; import * as uuidV4 from "uuid/v4"; @@ -148,6 +149,11 @@ async function getVersion(app: string): Promise { } export async function getCompressionMethod(): Promise { + // Disabling zstd on Windows due to https://github.com/actions/cache/issues/301 + if (os.platform() === "win32") { + return CompressionMethod.Gzip; + } + const versionOutput = await getVersion("zstd"); return versionOutput.toLowerCase().includes("zstd command line interface") ? CompressionMethod.Zstd From c8d75a8073ed6011e749c71d6d36ad091c71151b Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Sun, 10 May 2020 10:36:06 -0400 Subject: [PATCH 5/5] Use tries on all API calls --- __tests__/cacheHttpsClient.test.ts | 30 ++++--- dist/restore/index.js | 133 ++++++++++++++++++----------- dist/save/index.js | 133 ++++++++++++++++++----------- src/cacheHttpClient.ts | 50 ++++++----- 4 files changed, 214 insertions(+), 132 deletions(-) diff --git a/__tests__/cacheHttpsClient.test.ts b/__tests__/cacheHttpsClient.test.ts index 79e8210..da7a2d1 100644 --- a/__tests__/cacheHttpsClient.test.ts +++ b/__tests__/cacheHttpsClient.test.ts @@ -59,20 +59,17 @@ function handleResponse( async function testRetryExpectingResult( responses: Array, - expectedResult: string + expectedResult: string | null ): Promise { responses = responses.reverse(); // Reverse responses since we pop from end const actualResult = await retry( "test", () => handleResponse(responses.pop()), - (response: TestResponse) => response.statusCode, - (response: TestResponse) => response.result, - (statusCode: number) => statusCode === 200, - (statusCode: number) => statusCode === 503 + (response: TestResponse) => response.statusCode ); - expect(actualResult).toEqual(expectedResult); + expect(actualResult.result).toEqual(expectedResult); } async function testRetryExpectingError( @@ -84,10 +81,7 @@ async function testRetryExpectingError( retry( "test", () => handleResponse(responses.pop()), - (response: TestResponse) => response.statusCode, - (response: TestResponse) => response.result, - (statusCode: number) => statusCode === 200, - (statusCode: number) => statusCode === 503 + (response: TestResponse) => response.statusCode ) ).rejects.toBeInstanceOf(Error); } @@ -165,3 +159,19 @@ test("retry works after error", async () => { "Ok" ); }); + +test("retry returns after client error", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 400, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + null + ); +}); diff --git a/dist/restore/index.js b/dist/restore/index.js index d063a11..eaaeefc 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2197,6 +2197,12 @@ function isSuccessStatusCode(statusCode) { } return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode) { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} function isRetryableStatusCode(statusCode) { if (!statusCode) { return false; @@ -2246,7 +2252,7 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; -function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { +function retry(name, method, getStatusCode, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { let response = undefined; let statusCode = undefined; @@ -2257,8 +2263,8 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, try { response = yield method(); statusCode = getStatusCode(response); - if (isSuccessStatusCode(statusCode)) { - return getReturnValue(response); + if (!isServerErrorStatusCode(statusCode)) { + return response; } isRetryable = isRetryableStatusCode(statusCode); errorMessage = `Cache service responded with ${statusCode}`; @@ -2280,13 +2286,13 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, exports.retry = retry; function retryTypedResponse(name, method, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { - return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + return yield retry(name, method, (response) => response.statusCode, maxAttempts); }); } exports.retryTypedResponse = retryTypedResponse; function retryHttpClientResponse(name, method, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { - return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + return yield retry(name, method, (response) => response.message.statusCode, maxAttempts); }); } exports.retryHttpClientResponse = retryHttpClientResponse; @@ -2322,7 +2328,7 @@ function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); - const downloadResponse = yield httpClient.get(archiveLocation); + const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation)); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { downloadResponse.message.destroy(); @@ -2354,7 +2360,7 @@ function reserveCache(key, options) { key, version }; - const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest); + const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest)); return _d = (_c = (_b = response) === null || _b === void 0 ? void 0 : _b.result) === null || _c === void 0 ? void 0 : _c.cacheId, (_d !== null && _d !== void 0 ? _d : -1); }); } @@ -2427,7 +2433,7 @@ function uploadFile(httpClient, cacheId, archivePath) { function commitCache(httpClient, cacheId, filesize) { return __awaiter(this, void 0, void 0, function* () { const commitCacheRequest = { size: filesize }; - return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest); + return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest)); }); } function saveCache(cacheId, archivePath) { @@ -2469,7 +2475,9 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2505,7 +2513,8 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -3231,8 +3240,8 @@ const core = __importStar(__webpack_require__(470)); const exec = __importStar(__webpack_require__(986)); const glob = __importStar(__webpack_require__(281)); const io = __importStar(__webpack_require__(1)); -const os = __importStar(__webpack_require__(87)); const fs = __importStar(__webpack_require__(747)); +const os = __importStar(__webpack_require__(87)); const path = __importStar(__webpack_require__(622)); const util = __importStar(__webpack_require__(669)); const uuidV4 = __importStar(__webpack_require__(826)); @@ -3637,6 +3646,7 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; + HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -3661,8 +3671,18 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; -const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; +const HttpRedirectCodes = [ + HttpCodes.MovedPermanently, + HttpCodes.ResourceMoved, + HttpCodes.SeeOther, + HttpCodes.TemporaryRedirect, + HttpCodes.PermanentRedirect +]; +const HttpResponseRetryCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout +]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -3676,12 +3696,6 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); - this.message.on('aborted', () => { - reject("Request was aborted or closed prematurely"); - }); - this.message.on('timeout', (socket) => { - reject("Request timed out"); - }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3793,19 +3807,22 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error("Client has already been disposed."); + throw new Error('Client has already been disposed.'); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; + let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 + ? this._maxRetries + 1 + : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); - // Check if it's an authentication challenge - if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { + if (response && + response.message && + response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -3823,21 +3840,32 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 - && this._allowRedirects - && redirectsRemaining > 0) { - const redirectUrl = response.message.headers["location"]; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && + this._allowRedirects && + redirectsRemaining > 0) { + const redirectUrl = response.message.headers['location']; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { - throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); + if (parsedUrl.protocol == 'https:' && + parsedUrl.protocol != parsedRedirectUrl.protocol && + !this._allowRedirectDowngrade) { + throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); + // strip authorization header if redirected to a different hostname + if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { + for (let header in headers) { + // header names are case insensitive + if (header.toLowerCase() === 'authorization') { + delete headers[header]; + } + } + } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -3888,8 +3916,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof (data) === 'string') { - info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); + if (typeof data === 'string') { + info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -3902,7 +3930,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', (sock) => { + req.on('socket', sock => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -3915,13 +3943,12 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers - console.log(`Caught error on request: ${err}`); handleResult(err, null); }); - if (data && typeof (data) === 'string') { + if (data && typeof data === 'string') { req.write(data, 'utf8'); } - if (data && typeof (data) !== 'string') { + if (data && typeof data !== 'string') { data.on('close', function () { req.end(); }); @@ -3948,31 +3975,34 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; - info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port + ? parseInt(info.parsedUrl.port) + : defaultPort; + info.options.path = + (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers["user-agent"] = this.userAgent; + info.options.headers['user-agent'] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach((handler) => { + this.handlers.forEach(handler => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -4010,7 +4040,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - }, + } }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -4037,7 +4067,9 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); + agent.options = Object.assign(agent.options || {}, { + rejectUnauthorized: false + }); } return agent; } @@ -4098,7 +4130,7 @@ class HttpClient { msg = contents; } else { - msg = "Failed request: (" + statusCode + ")"; + msg = 'Failed request: (' + statusCode + ')'; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -5187,12 +5219,10 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env["https_proxy"] || - process.env["HTTPS_PROXY"]; + proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; } else { - proxyVar = process.env["http_proxy"] || - process.env["HTTP_PROXY"]; + proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -5204,7 +5234,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; + let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; if (!noProxy) { return false; } @@ -5225,7 +5255,10 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { + for (let upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/dist/save/index.js b/dist/save/index.js index 7dba8da..6e6d618 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2197,6 +2197,12 @@ function isSuccessStatusCode(statusCode) { } return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode) { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} function isRetryableStatusCode(statusCode) { if (!statusCode) { return false; @@ -2246,7 +2252,7 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; -function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { +function retry(name, method, getStatusCode, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { let response = undefined; let statusCode = undefined; @@ -2257,8 +2263,8 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, try { response = yield method(); statusCode = getStatusCode(response); - if (isSuccessStatusCode(statusCode)) { - return getReturnValue(response); + if (!isServerErrorStatusCode(statusCode)) { + return response; } isRetryable = isRetryableStatusCode(statusCode); errorMessage = `Cache service responded with ${statusCode}`; @@ -2280,13 +2286,13 @@ function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, exports.retry = retry; function retryTypedResponse(name, method, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { - return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + return yield retry(name, method, (response) => response.statusCode, maxAttempts); }); } exports.retryTypedResponse = retryTypedResponse; function retryHttpClientResponse(name, method, maxAttempts = 2) { return __awaiter(this, void 0, void 0, function* () { - return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + return yield retry(name, method, (response) => response.message.statusCode, maxAttempts); }); } exports.retryHttpClientResponse = retryHttpClientResponse; @@ -2322,7 +2328,7 @@ function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); - const downloadResponse = yield httpClient.get(archiveLocation); + const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation)); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { downloadResponse.message.destroy(); @@ -2354,7 +2360,7 @@ function reserveCache(key, options) { key, version }; - const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest); + const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest)); return _d = (_c = (_b = response) === null || _b === void 0 ? void 0 : _b.result) === null || _c === void 0 ? void 0 : _c.cacheId, (_d !== null && _d !== void 0 ? _d : -1); }); } @@ -2427,7 +2433,7 @@ function uploadFile(httpClient, cacheId, archivePath) { function commitCache(httpClient, cacheId, filesize) { return __awaiter(this, void 0, void 0, function* () { const commitCacheRequest = { size: filesize }; - return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest); + return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest)); }); } function saveCache(cacheId, archivePath) { @@ -2469,7 +2475,9 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2505,7 +2513,8 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -3231,8 +3240,8 @@ const core = __importStar(__webpack_require__(470)); const exec = __importStar(__webpack_require__(986)); const glob = __importStar(__webpack_require__(281)); const io = __importStar(__webpack_require__(1)); -const os = __importStar(__webpack_require__(87)); const fs = __importStar(__webpack_require__(747)); +const os = __importStar(__webpack_require__(87)); const path = __importStar(__webpack_require__(622)); const util = __importStar(__webpack_require__(669)); const uuidV4 = __importStar(__webpack_require__(826)); @@ -3637,6 +3646,7 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; + HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -3661,8 +3671,18 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; -const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; +const HttpRedirectCodes = [ + HttpCodes.MovedPermanently, + HttpCodes.ResourceMoved, + HttpCodes.SeeOther, + HttpCodes.TemporaryRedirect, + HttpCodes.PermanentRedirect +]; +const HttpResponseRetryCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout +]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -3676,12 +3696,6 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); - this.message.on('aborted', () => { - reject("Request was aborted or closed prematurely"); - }); - this.message.on('timeout', (socket) => { - reject("Request timed out"); - }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3793,19 +3807,22 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error("Client has already been disposed."); + throw new Error('Client has already been disposed.'); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; + let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 + ? this._maxRetries + 1 + : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); - // Check if it's an authentication challenge - if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { + if (response && + response.message && + response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -3823,21 +3840,32 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 - && this._allowRedirects - && redirectsRemaining > 0) { - const redirectUrl = response.message.headers["location"]; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && + this._allowRedirects && + redirectsRemaining > 0) { + const redirectUrl = response.message.headers['location']; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { - throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); + if (parsedUrl.protocol == 'https:' && + parsedUrl.protocol != parsedRedirectUrl.protocol && + !this._allowRedirectDowngrade) { + throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); + // strip authorization header if redirected to a different hostname + if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { + for (let header in headers) { + // header names are case insensitive + if (header.toLowerCase() === 'authorization') { + delete headers[header]; + } + } + } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -3888,8 +3916,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof (data) === 'string') { - info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); + if (typeof data === 'string') { + info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -3902,7 +3930,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', (sock) => { + req.on('socket', sock => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -3915,13 +3943,12 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers - console.log(`Caught error on request: ${err}`); handleResult(err, null); }); - if (data && typeof (data) === 'string') { + if (data && typeof data === 'string') { req.write(data, 'utf8'); } - if (data && typeof (data) !== 'string') { + if (data && typeof data !== 'string') { data.on('close', function () { req.end(); }); @@ -3948,31 +3975,34 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; - info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port + ? parseInt(info.parsedUrl.port) + : defaultPort; + info.options.path = + (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers["user-agent"] = this.userAgent; + info.options.headers['user-agent'] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach((handler) => { + this.handlers.forEach(handler => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -4010,7 +4040,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - }, + } }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -4037,7 +4067,9 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); + agent.options = Object.assign(agent.options || {}, { + rejectUnauthorized: false + }); } return agent; } @@ -4098,7 +4130,7 @@ class HttpClient { msg = contents; } else { - msg = "Failed request: (" + statusCode + ")"; + msg = 'Failed request: (' + statusCode + ')'; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -5164,12 +5196,10 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env["https_proxy"] || - process.env["HTTPS_PROXY"]; + proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; } else { - proxyVar = process.env["http_proxy"] || - process.env["HTTP_PROXY"]; + proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -5181,7 +5211,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; + let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; if (!noProxy) { return false; } @@ -5202,7 +5232,10 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { + for (let upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 7f06b6b..d7d50ce 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -30,6 +30,13 @@ function isSuccessStatusCode(statusCode?: number): boolean { return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode?: number): boolean { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} + function isRetryableStatusCode(statusCode?: number): boolean { if (!statusCode) { return false; @@ -99,16 +106,13 @@ export function getCacheVersion(compressionMethod?: CompressionMethod): string { .digest("hex"); } -export async function retry( +export async function retry( name: string, - method: () => Promise, - getStatusCode: (R) => number | undefined, - getReturnValue: (R) => T, - isSuccessStatusCode: (number) => boolean, - isRetryableStatusCode: (number) => boolean, + method: () => Promise, + getStatusCode: (T) => number | undefined, maxAttempts = 2 ): Promise { - let response: R | undefined = undefined; + let response: T | undefined = undefined; let statusCode: number | undefined = undefined; let isRetryable = false; let errorMessage = ""; @@ -119,8 +123,8 @@ export async function retry( response = await method(); statusCode = getStatusCode(response); - if (isSuccessStatusCode(statusCode)) { - return getReturnValue(response); + if (!isServerErrorStatusCode(statusCode)) { + return response; } isRetryable = isRetryableStatusCode(statusCode); @@ -154,9 +158,6 @@ export async function retryTypedResponse( name, method, (response: ITypedResponse) => response.statusCode, - (response: ITypedResponse) => response, - isSuccessStatusCode, - isRetryableStatusCode, maxAttempts ); } @@ -170,9 +171,6 @@ export async function retryHttpClientResponse( name, method, (response: IHttpClientResponse) => response.message.statusCode, - (response: IHttpClientResponse) => response, - isSuccessStatusCode, - isRetryableStatusCode, maxAttempts ); } @@ -221,7 +219,10 @@ export async function downloadCache( ): Promise { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient("actions/cache"); - const downloadResponse = await httpClient.get(archiveLocation); + const downloadResponse = await retryHttpClientResponse( + "downloadCache", + () => httpClient.get(archiveLocation) + ); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(SocketTimeout, () => { @@ -263,10 +264,13 @@ export async function reserveCache( key, version }; - const response = await httpClient.postJson( - getCacheApiUrl("caches"), - reserveCacheRequest + const response = await retryTypedResponse("reserveCache", () => + httpClient.postJson( + getCacheApiUrl("caches"), + reserveCacheRequest + ) ); + return response?.result?.cacheId ?? -1; } @@ -381,9 +385,11 @@ async function commitCache( filesize: number ): Promise> { const commitCacheRequest: CommitCacheRequest = { size: filesize }; - return await httpClient.postJson( - getCacheApiUrl(`caches/${cacheId.toString()}`), - commitCacheRequest + return await retryTypedResponse("commitCache", () => + httpClient.postJson( + getCacheApiUrl(`caches/${cacheId.toString()}`), + commitCacheRequest + ) ); }