From bad827c28e0951618afe64f8c9a46268c3dbf707 Mon Sep 17 00:00:00 2001 From: Josh Gross Date: Fri, 13 Dec 2019 15:19:25 -0500 Subject: [PATCH] Initial pass at chunked upload apis --- __tests__/save.test.ts | 4 +- package-lock.json | 12 +-- package.json | 6 +- src/cacheHttpClient.ts | 167 ++++++++++++++++++++++++++++++++--------- src/contracts.d.ts | 13 ++++ src/save.ts | 6 +- 6 files changed, 158 insertions(+), 50 deletions(-) diff --git a/__tests__/save.test.ts b/__tests__/save.test.ts index 89657c4..b0eb462 100644 --- a/__tests__/save.test.ts +++ b/__tests__/save.test.ts @@ -200,7 +200,7 @@ test("save with large cache outputs warning", async () => { const execMock = jest.spyOn(exec, "exec"); - const cacheSize = 1024 * 1024 * 1024; //~1GB, over the 400MB limit + const cacheSize = 4 * 1024 * 1024 * 1024; //~4GB, over the 2GB limit jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => { return cacheSize; }); @@ -227,7 +227,7 @@ test("save with large cache outputs warning", async () => { expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledWith( - "Cache size of ~1024 MB (1073741824 B) is over the 400MB limit, not saving cache." + "Cache size of ~4 GB (4294967296 B) is over the 2GB limit, not saving cache." ); expect(failedMock).toHaveBeenCalledTimes(0); diff --git a/package-lock.json b/package-lock.json index 2e8413e..986b08b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4859,9 +4859,9 @@ "dev": true }, "prettier": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz", - "integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", "dev": true }, "prettier-linter-helpers": { @@ -5983,9 +5983,9 @@ } }, "typescript": { - "version": "3.6.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.4.tgz", - "integrity": "sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz", + "integrity": "sha512-Mcr/Qk7hXqFBXMN7p7Lusj1ktCBydylfQM/FZCk5glCNQJrCUKPkMHdo9R0MTFWsC/4kPFvDS0fDPvukfCkFsw==", "dev": true }, "uglify-js": { diff --git a/package.json b/package.json index 42fbdbe..7de321b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cache", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "Cache dependencies and build outputs", "main": "dist/restore/index.js", @@ -46,8 +46,8 @@ "jest": "^24.8.0", "jest-circus": "^24.7.1", "nock": "^11.7.0", - "prettier": "1.18.2", + "prettier": "^1.19.1", "ts-jest": "^24.0.2", - "typescript": "^3.6.4" + "typescript": "^3.7.3" } } diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 8a2014f..7eee046 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -3,24 +3,39 @@ import * as fs from "fs"; import { BearerCredentialHandler } from "typed-rest-client/Handlers"; import { HttpClient } from "typed-rest-client/HttpClient"; import { IHttpClientResponse } from "typed-rest-client/Interfaces"; -import { IRequestOptions, RestClient } from "typed-rest-client/RestClient"; -import { ArtifactCacheEntry } from "./contracts"; +import { + IRequestOptions, + RestClient, + IRestResponse +} from "typed-rest-client/RestClient"; +import { + ArtifactCacheEntry, + CommitCacheRequest, + ReserveCacheRequest, + ReserverCacheResponse +} from "./contracts"; +import * as utils from "./utils/actionUtils"; -function getCacheUrl(): string { +const MAX_CHUNK_SIZE = 4000000; // 4 MB Chunks + +function isSuccessStatusCode(statusCode: number): boolean { + return statusCode >= 200 && statusCode < 300; +} +function getCacheApiUrl(): string { // Ideally we just use ACTIONS_CACHE_URL - const cacheUrl: string = ( + const baseUrl: string = ( process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_RUNTIME_URL"] || "" ).replace("pipelines", "artifactcache"); - if (!cacheUrl) { + if (!baseUrl) { throw new Error( "Cache Service Url not found, unable to restore cache." ); } - core.debug(`Cache Url: ${cacheUrl}`); - return cacheUrl; + core.debug(`Cache Url: ${baseUrl}`); + return `${baseUrl}_apis/artifactcache/`; } function createAcceptHeader(type: string, apiVersion: string): string { @@ -29,7 +44,7 @@ function createAcceptHeader(type: string, apiVersion: string): string { function getRequestOptions(): IRequestOptions { const requestOptions: IRequestOptions = { - acceptHeader: createAcceptHeader("application/json", "5.2-preview.1") + acceptHeader: createAcceptHeader("application/json", "6.0-preview.1") }; return requestOptions; @@ -38,13 +53,11 @@ function getRequestOptions(): IRequestOptions { export async function getCacheEntry( keys: string[] ): Promise { - const cacheUrl = getCacheUrl(); + const cacheUrl = getCacheApiUrl(); const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const bearerCredentialHandler = new BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent( - keys.join(",") - )}`; + const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; const restClient = new RestClient("actions/cache", cacheUrl, [ bearerCredentialHandler @@ -57,14 +70,15 @@ export async function getCacheEntry( if (response.statusCode === 204) { return null; } - if (response.statusCode !== 200) { + if (!isSuccessStatusCode(response.statusCode)) { throw new Error(`Cache service responded with ${response.statusCode}`); } const cacheResult = response.result; - if (!cacheResult || !cacheResult.archiveLocation) { + const cacheDownloadUrl = cacheResult?.archiveLocation; + if (!cacheDownloadUrl) { throw new Error("Cache not found."); } - core.setSecret(cacheResult.archiveLocation); + core.setSecret(cacheDownloadUrl); core.debug(`Cache Result:`); core.debug(JSON.stringify(cacheResult)); @@ -83,46 +97,127 @@ async function pipeResponseToStream( } export async function downloadCache( - cacheEntry: ArtifactCacheEntry, + archiveLocation: string, archivePath: string ): Promise { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient("actions/cache"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const downloadResponse = await httpClient.get(cacheEntry.archiveLocation!); + const downloadResponse = await httpClient.get(archiveLocation); await pipeResponseToStream(downloadResponse, stream); } +// Returns Cache ID +async function reserveCache( + restClient: RestClient, + key: string +): Promise { + const reserveCacheRequest: ReserveCacheRequest = { + key + }; + const response = await restClient.create( + "caches", + reserveCacheRequest + ); + + return response?.result?.cacheId || -1; +} + +function getContentRange(start: number, length: number): string { + // Format: `bytes start-end/filesize + // start and end are inclusive + // filesize can be * + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/* + return `bytes ${start}-${start + length - 1}/*`; +} + +async function uploadChunk( + restClient: RestClient, + cacheId: number, + data: Buffer, + offset: number +): Promise> { + const requestOptions = getRequestOptions(); + requestOptions.additionalHeaders = { + "Content-Type": "application/octet-stream", + "Content-Range": getContentRange(offset, data.byteLength) + }; + + return await restClient.update( + cacheId.toString(), + data.toString("utf8"), + requestOptions + ); +} + +async function commitCache( + restClient: RestClient, + cacheId: number, + filesize: number +): Promise> { + const requestOptions = getRequestOptions(); + const commitCacheRequest: CommitCacheRequest = { size: filesize }; + return await restClient.create( + cacheId.toString(), + commitCacheRequest, + requestOptions + ); +} + export async function saveCache( key: string, archivePath: string ): Promise { - const stream = fs.createReadStream(archivePath); - - const cacheUrl = getCacheUrl(); const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const bearerCredentialHandler = new BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`; - const postUrl = cacheUrl + resource; - - const restClient = new RestClient("actions/cache", undefined, [ + const restClient = new RestClient("actions/cache", getCacheApiUrl(), [ bearerCredentialHandler ]); - const requestOptions = getRequestOptions(); - requestOptions.additionalHeaders = { - "Content-Type": "application/octet-stream" - }; + // Reserve Cache + const cacheId = await reserveCache(restClient, key); + if (cacheId < 0) { + throw new Error(`Unable to reserve cache.`); + } - const response = await restClient.uploadStream( - "POST", - postUrl, - stream, - requestOptions + // Upload Chunks + const stream = fs.createReadStream(archivePath); + let streamIsClosed = false; + stream.on("close", () => { + streamIsClosed = true; + }); + + const uploads: Promise>[] = []; + let offset = 0; + while (!streamIsClosed) { + const chunk: Buffer = stream.read(MAX_CHUNK_SIZE); + uploads.push(uploadChunk(restClient, cacheId, chunk, offset)); + offset += MAX_CHUNK_SIZE; + } + + const responses = await Promise.all(uploads); + + const failedResponse = responses.find( + x => !isSuccessStatusCode(x.statusCode) ); - if (response.statusCode !== 200) { - throw new Error(`Cache service responded with ${response.statusCode}`); + if (failedResponse) { + throw new Error( + `Cache service responded with ${failedResponse.statusCode} during chunk upload.` + ); + } + + // Commit Cache + const cacheSize = utils.getArchiveFileSize(archivePath); + const commitCacheResponse = await commitCache( + restClient, + cacheId, + cacheSize + ); + if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { + throw new Error( + `Cache service responded with ${commitCacheResponse.statusCode} during commit cache.` + ); } core.info("Cache saved successfully"); diff --git a/src/contracts.d.ts b/src/contracts.d.ts index 8478b83..4bc7979 100644 --- a/src/contracts.d.ts +++ b/src/contracts.d.ts @@ -4,3 +4,16 @@ export interface ArtifactCacheEntry { creationTime?: string; archiveLocation?: string; } + +export interface CommitCacheRequest { + size: number; +} + +export interface ReserveCacheRequest { + key: string; + version?: string; +} + +export interface ReserverCacheResponse { + cacheId: number; +} diff --git a/src/save.ts b/src/save.ts index 21f32d3..51c82e0 100644 --- a/src/save.ts +++ b/src/save.ts @@ -65,14 +65,14 @@ async function run(): Promise { core.debug(`Tar Path: ${tarPath}`); await exec(`"${tarPath}"`, args); - const fileSizeLimit = 400 * 1024 * 1024; // 400MB + const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit const archiveFileSize = utils.getArchiveFileSize(archivePath); core.debug(`File Size: ${archiveFileSize}`); if (archiveFileSize > fileSizeLimit) { utils.logWarning( `Cache size of ~${Math.round( - archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.` + archiveFileSize / (1024 * 1024 * 1024) + )} GB (${archiveFileSize} B) is over the 2GB limit, not saving cache.` ); return; }