diff --git a/__tests__/cacheHttpsClient.test.ts b/__tests__/cacheHttpsClient.test.ts index 362beb4..79e8210 100644 --- a/__tests__/cacheHttpsClient.test.ts +++ b/__tests__/cacheHttpsClient.test.ts @@ -1,4 +1,4 @@ -import { getCacheVersion } from "../src/cacheHttpClient"; +import { getCacheVersion, retry } from "../src/cacheHttpClient"; import { CompressionMethod, Inputs } from "../src/constants"; import * as testUtils from "../src/utils/testUtils"; @@ -37,3 +37,131 @@ test("getCacheVersion with gzip compression does not change vesion", async () => test("getCacheVersion with no input throws", async () => { expect(() => getCacheVersion()).toThrow(); }); + +interface TestResponse { + statusCode: number; + result: string | null; +} + +function handleResponse( + response: TestResponse | undefined +): Promise { + if (!response) { + fail("Retry method called too many times"); + } + + if (response.statusCode === 999) { + throw Error("Test Error"); + } else { + return Promise.resolve(response); + } +} + +async function testRetryExpectingResult( + responses: Array, + expectedResult: string +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + const actualResult = await retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode, + (response: TestResponse) => response.result, + (statusCode: number) => statusCode === 200, + (statusCode: number) => statusCode === 503 + ); + + expect(actualResult).toEqual(expectedResult); +} + +async function testRetryExpectingError( + responses: Array +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + expect( + retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode, + (response: TestResponse) => response.result, + (statusCode: number) => statusCode === 200, + (statusCode: number) => statusCode === 503 + ) + ).rejects.toBeInstanceOf(Error); +} + +test("retry works on successful response", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry works after retryable status code", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry fails after exhausting retries", async () => { + await testRetryExpectingError([ + { + statusCode: 503, + result: null + }, + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry fails after non-retryable status code", async () => { + await testRetryExpectingError([ + { + statusCode: 500, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry works after error", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 999, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); diff --git a/dist/restore/index.js b/dist/restore/index.js index 67707ad..f0b155d 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2246,19 +2246,60 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; +function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys, options) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const version = getCacheVersion((_a = options) === null || _a === void 0 ? void 0 : _a.compressionMethod); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}&version=${version}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = (_b = cacheResult) === null || _b === void 0 ? void 0 : _b.archiveLocation; if (!cacheDownloadUrl) { @@ -2326,7 +2367,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -2336,20 +2377,9 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, uploadChunkRequest); }); } function parseEnvNumber(key) { @@ -2379,13 +2409,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -3642,6 +3671,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3763,6 +3798,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -3874,6 +3910,7 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers + console.log(`Caught error on request: ${err}`); handleResult(err, null); }); if (data && typeof (data) === 'string') { diff --git a/dist/save/index.js b/dist/save/index.js index 3d7a6df..6dd837c 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2246,19 +2246,60 @@ function getCacheVersion(compressionMethod) { .digest("hex"); } exports.getCacheVersion = getCacheVersion; +function retry(name, method, getStatusCode, getReturnValue, isSuccessStatusCode, isRetryableStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, (response) => response, isSuccessStatusCode, isRetryableStatusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys, options) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const version = getCacheVersion((_a = options) === null || _a === void 0 ? void 0 : _a.compressionMethod); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}&version=${version}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = (_b = cacheResult) === null || _b === void 0 ? void 0 : _b.archiveLocation; if (!cacheDownloadUrl) { @@ -2326,7 +2367,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -2336,20 +2377,9 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, uploadChunkRequest); }); } function parseEnvNumber(key) { @@ -2379,13 +2409,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -3642,6 +3671,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -3763,6 +3798,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -3874,6 +3910,7 @@ class HttpClient { req.on('error', function (err) { // err has statusCode property // res should have headers + console.log(`Caught error on request: ${err}`); handleResult(err, null); }); if (data && typeof (data) === 'string') { diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index c000b7f..7f06b6b 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -99,6 +99,84 @@ export function getCacheVersion(compressionMethod?: CompressionMethod): string { .digest("hex"); } +export async function retry( + name: string, + method: () => Promise, + getStatusCode: (R) => number | undefined, + getReturnValue: (R) => T, + isSuccessStatusCode: (number) => boolean, + isRetryableStatusCode: (number) => boolean, + maxAttempts = 2 +): Promise { + let response: R | undefined = undefined; + let statusCode: number | undefined = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + + while (attempt <= maxAttempts) { + try { + response = await method(); + statusCode = getStatusCode(response); + + if (isSuccessStatusCode(statusCode)) { + return getReturnValue(response); + } + + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } catch (error) { + isRetryable = true; + errorMessage = error.message; + } + + core.debug( + `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}` + ); + + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + + attempt++; + } + + throw Error(`${name} failed: ${errorMessage}`); +} + +export async function retryTypedResponse( + name: string, + method: () => Promise>, + maxAttempts = 2 +): Promise> { + return await retry( + name, + method, + (response: ITypedResponse) => response.statusCode, + (response: ITypedResponse) => response, + isSuccessStatusCode, + isRetryableStatusCode, + maxAttempts + ); +} + +export async function retryHttpClientResponse( + name: string, + method: () => Promise, + maxAttempts = 2 +): Promise { + return await retry( + name, + method, + (response: IHttpClientResponse) => response.message.statusCode, + (response: IHttpClientResponse) => response, + isSuccessStatusCode, + isRetryableStatusCode, + maxAttempts + ); +} + export async function getCacheEntry( keys: string[], options?: CacheOptions @@ -109,15 +187,13 @@ export async function getCacheEntry( keys.join(",") )}&version=${version}`; - const response = await httpClient.getJson( - getCacheApiUrl(resource) + const response = await retryTypedResponse("getCacheEntry", () => + httpClient.getJson(getCacheApiUrl(resource)) ); + if (response.statusCode === 204) { return null; } - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`Cache service responded with ${response.statusCode}`); - } const cacheResult = response.result; const cacheDownloadUrl = cacheResult?.archiveLocation; @@ -206,7 +282,7 @@ function getContentRange(start: number, end: number): string { async function uploadChunk( httpClient: HttpClient, resourceUrl: string, - data: NodeJS.ReadableStream, + openStream: () => NodeJS.ReadableStream, start: number, end: number ): Promise { @@ -227,28 +303,14 @@ async function uploadChunk( return await httpClient.sendStream( "PATCH", resourceUrl, - data, + openStream(), additionalHeaders ); }; - const response = await uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug( - `Received ${response.message.statusCode}, retrying chunk at offset ${start}.` - ); - const retryResponse = await uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - - throw new Error( - `Cache service responded with ${response.message.statusCode} during chunk upload.` + await retryHttpClientResponse( + `uploadChunk (start: ${start}, end: ${end})`, + uploadChunkRequest ); } @@ -290,17 +352,17 @@ async function uploadFile( const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }); await uploadChunk( httpClient, resourceUrl, - chunk, + () => + fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end );