From 4c4974aff182a0ee174fefeba291c7741b0dce89 Mon Sep 17 00:00:00 2001 From: Josh Gross Date: Mon, 6 Jan 2020 13:36:33 -0500 Subject: [PATCH] Release v1.1 --- README.md | 27 +++- __tests__/restore.test.ts | 87 ++++-------- __tests__/save.test.ts | 152 ++++++++++++--------- __tests__/tar.test.ts | 58 ++++++++ dist/restore/index.js | 265 ++++++++++++++++++++++++++++-------- dist/save/index.js | 273 ++++++++++++++++++++++++++++++-------- examples.md | 58 +++++--- package-lock.json | 26 ++-- package.json | 6 +- src/cacheHttpClient.ts | 250 +++++++++++++++++++++++++++------- src/contracts.d.ts | 13 ++ src/restore.ts | 32 +---- src/save.ts | 38 +++--- src/tar.ts | 47 +++++++ 14 files changed, 959 insertions(+), 373 deletions(-) create mode 100644 __tests__/tar.test.ts create mode 100644 src/tar.ts diff --git a/README.md b/README.md index 1278fae..02cf6fe 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ on: push jobs: build: runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v1 @@ -49,14 +49,31 @@ jobs: - name: Generate Prime Numbers if: steps.cache-primes.outputs.cache-hit != 'true' run: /generate-primes.sh -d prime-numbers - + - name: Use Prime Numbers run: /primes.sh -d prime-numbers ``` -## Ecosystem Examples +## Implementation Examples + +Every programming language and framework has its own way of caching. + +See [Examples](examples.md) for a list of `actions/cache` implementations for use with: + +- [C# - Nuget](./examples.md#c---nuget) +- [Elixir - Mix](./examples.md#elixir---mix) +- [Go - Modules](./examples.md#go---modules) +- [Java - Gradle](./examples.md#java---gradle) +- [Java - Maven](./examples.md#java---maven) +- [Node - npm](./examples.md#node---npm) +- [Node - Yarn](./examples.md#node---yarn) +- [PHP - Composer](./examples.md#php---composer) +- [Python - pip](./examples.md#python---pip) +- [Ruby - Gem](./examples.md#ruby---gem) +- [Rust - Cargo](./examples.md#rust---cargo) +- [Swift, Objective-C - Carthage](./examples.md#swift-objective-c---carthage) +- [Swift, Objective-C - CocoaPods](./examples.md#swift-objective-c---cocoapods) -See [Examples](examples.md) ## Cache Limits @@ -76,7 +93,7 @@ steps: with: path: path/to/dependencies key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }} - + - name: Install Dependencies if: steps.cache.outputs.cache-hit != 'true' run: /install.sh diff --git a/__tests__/restore.test.ts b/__tests__/restore.test.ts index 78b00ec..c96a2d6 100644 --- a/__tests__/restore.test.ts +++ b/__tests__/restore.test.ts @@ -1,18 +1,16 @@ import * as core from "@actions/core"; -import * as exec from "@actions/exec"; -import * as io from "@actions/io"; import * as path from "path"; import * as cacheHttpClient from "../src/cacheHttpClient"; import { Events, Inputs } from "../src/constants"; import { ArtifactCacheEntry } from "../src/contracts"; import run from "../src/restore"; +import * as tar from "../src/tar"; import * as actionUtils from "../src/utils/actionUtils"; import * as testUtils from "../src/utils/testUtils"; -jest.mock("@actions/exec"); -jest.mock("@actions/io"); -jest.mock("../src/utils/actionUtils"); jest.mock("../src/cacheHttpClient"); +jest.mock("../src/tar"); +jest.mock("../src/utils/actionUtils"); beforeAll(() => { jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => { @@ -35,10 +33,6 @@ beforeAll(() => { const actualUtils = jest.requireActual("../src/utils/actionUtils"); return actualUtils.getSupportedEvents(); }); - - jest.spyOn(io, "which").mockImplementation(tool => { - return Promise.resolve(tool); - }); }); beforeEach(() => { @@ -245,8 +239,7 @@ test("restore with cache found", async () => { .spyOn(actionUtils, "getArchiveFileSize") .mockReturnValue(fileSize); - const mkdirMock = jest.spyOn(io, "mkdirP"); - const execMock = jest.spyOn(exec, "exec"); + const extractTarMock = jest.spyOn(tar, "extractTar"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); await run(); @@ -255,24 +248,14 @@ test("restore with cache found", async () => { expect(getCacheMock).toHaveBeenCalledWith([key]); expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry); expect(createTempDirectoryMock).toHaveBeenCalledTimes(1); - expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath); + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath + ); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); - expect(mkdirMock).toHaveBeenCalledWith(cachePath); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-xz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/") - ] - : ["-xz", "-f", archivePath, "-C", cachePath]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(extractTarMock).toHaveBeenCalledTimes(1); + expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith(true); @@ -323,8 +306,7 @@ test("restore with a pull request event and cache found", async () => { .spyOn(actionUtils, "getArchiveFileSize") .mockReturnValue(fileSize); - const mkdirMock = jest.spyOn(io, "mkdirP"); - const execMock = jest.spyOn(exec, "exec"); + const extractTarMock = jest.spyOn(tar, "extractTar"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); await run(); @@ -333,25 +315,15 @@ test("restore with a pull request event and cache found", async () => { expect(getCacheMock).toHaveBeenCalledWith([key]); expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry); expect(createTempDirectoryMock).toHaveBeenCalledTimes(1); - expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath); + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath + ); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`); - expect(mkdirMock).toHaveBeenCalledWith(cachePath); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-xz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/") - ] - : ["-xz", "-f", archivePath, "-C", cachePath]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(extractTarMock).toHaveBeenCalledTimes(1); + expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith(true); @@ -402,8 +374,7 @@ test("restore with cache found for restore key", async () => { .spyOn(actionUtils, "getArchiveFileSize") .mockReturnValue(fileSize); - const mkdirMock = jest.spyOn(io, "mkdirP"); - const execMock = jest.spyOn(exec, "exec"); + const extractTarMock = jest.spyOn(tar, "extractTar"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); await run(); @@ -412,25 +383,15 @@ test("restore with cache found for restore key", async () => { expect(getCacheMock).toHaveBeenCalledWith([key, restoreKey]); expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry); expect(createTempDirectoryMock).toHaveBeenCalledTimes(1); - expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath); + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath + ); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`); - expect(mkdirMock).toHaveBeenCalledWith(cachePath); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-xz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/") - ] - : ["-xz", "-f", archivePath, "-C", cachePath]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(extractTarMock).toHaveBeenCalledTimes(1); + expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith(false); diff --git a/__tests__/save.test.ts b/__tests__/save.test.ts index 89657c4..b355076 100644 --- a/__tests__/save.test.ts +++ b/__tests__/save.test.ts @@ -1,19 +1,17 @@ import * as core from "@actions/core"; -import * as exec from "@actions/exec"; -import * as io from "@actions/io"; import * as path from "path"; import * as cacheHttpClient from "../src/cacheHttpClient"; import { Events, Inputs } from "../src/constants"; import { ArtifactCacheEntry } from "../src/contracts"; import run from "../src/save"; +import * as tar from "../src/tar"; import * as actionUtils from "../src/utils/actionUtils"; import * as testUtils from "../src/utils/testUtils"; jest.mock("@actions/core"); -jest.mock("@actions/exec"); -jest.mock("@actions/io"); -jest.mock("../src/utils/actionUtils"); jest.mock("../src/cacheHttpClient"); +jest.mock("../src/tar"); +jest.mock("../src/utils/actionUtils"); beforeAll(() => { jest.spyOn(core, "getInput").mockImplementation((name, options) => { @@ -49,10 +47,6 @@ beforeAll(() => { jest.spyOn(actionUtils, "createTempDirectory").mockImplementation(() => { return Promise.resolve("/foo/bar"); }); - - jest.spyOn(io, "which").mockImplementation(tool => { - return Promise.resolve(tool); - }); }); beforeEach(() => { @@ -128,7 +122,7 @@ test("save with exact match returns early", async () => { return primaryKey; }); - const execMock = jest.spyOn(exec, "exec"); + const createTarMock = jest.spyOn(tar, "createTar"); await run(); @@ -136,7 +130,7 @@ test("save with exact match returns early", async () => { `Cache hit occurred on the primary key ${primaryKey}, not saving cache.` ); - expect(execMock).toHaveBeenCalledTimes(0); + expect(createTarMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0); }); @@ -198,9 +192,9 @@ test("save with large cache outputs warning", async () => { const cachePath = path.resolve(inputPath); testUtils.setInput(Inputs.Path, inputPath); - const execMock = jest.spyOn(exec, "exec"); + const createTarMock = jest.spyOn(tar, "createTar"); - const cacheSize = 1024 * 1024 * 1024; //~1GB, over the 400MB limit + const cacheSize = 4 * 1024 * 1024 * 1024; //~4GB, over the 2GB limit jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => { return cacheSize; }); @@ -209,30 +203,68 @@ test("save with large cache outputs warning", async () => { const archivePath = path.join("/foo/bar", "cache.tgz"); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] - : ["-cz", "-f", archivePath, "-C", cachePath, "."]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(createTarMock).toHaveBeenCalledTimes(1); + expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledWith( - "Cache size of ~1024 MB (1073741824 B) is over the 400MB limit, not saving cache." + "Cache size of ~4096 MB (4294967296 B) is over the 2GB limit, not saving cache." ); expect(failedMock).toHaveBeenCalledTimes(0); }); +test("save with reserve cache failure outputs warning", async () => { + const infoMock = jest.spyOn(core, "info"); + const logWarningMock = jest.spyOn(actionUtils, "logWarning"); + const failedMock = jest.spyOn(core, "setFailed"); + + const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; + const cacheEntry: ArtifactCacheEntry = { + cacheKey: "Linux-node-", + scope: "refs/heads/master", + creationTime: "2019-11-13T19:18:02+00:00", + archiveLocation: "www.actionscache.test/download" + }; + + jest.spyOn(core, "getState") + // Cache Entry State + .mockImplementationOnce(() => { + return JSON.stringify(cacheEntry); + }) + // Cache Key State + .mockImplementationOnce(() => { + return primaryKey; + }); + + const inputPath = "node_modules"; + testUtils.setInput(Inputs.Path, inputPath); + + const reserveCacheMock = jest + .spyOn(cacheHttpClient, "reserveCache") + .mockImplementationOnce(() => { + return Promise.resolve(-1); + }); + + const createTarMock = jest.spyOn(tar, "createTar"); + + const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); + + await run(); + + expect(reserveCacheMock).toHaveBeenCalledTimes(1); + expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); + + expect(infoMock).toHaveBeenCalledWith( + `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.` + ); + + expect(createTarMock).toHaveBeenCalledTimes(0); + expect(saveCacheMock).toHaveBeenCalledTimes(0); + expect(logWarningMock).toHaveBeenCalledTimes(0); + expect(failedMock).toHaveBeenCalledTimes(0); +}); + test("save with server error outputs warning", async () => { const logWarningMock = jest.spyOn(actionUtils, "logWarning"); const failedMock = jest.spyOn(core, "setFailed"); @@ -259,7 +291,14 @@ test("save with server error outputs warning", async () => { const cachePath = path.resolve(inputPath); testUtils.setInput(Inputs.Path, inputPath); - const execMock = jest.spyOn(exec, "exec"); + const cacheId = 4; + const reserveCacheMock = jest + .spyOn(cacheHttpClient, "reserveCache") + .mockImplementationOnce(() => { + return Promise.resolve(cacheId); + }); + + const createTarMock = jest.spyOn(tar, "createTar"); const saveCacheMock = jest .spyOn(cacheHttpClient, "saveCache") @@ -269,26 +308,16 @@ test("save with server error outputs warning", async () => { await run(); + expect(reserveCacheMock).toHaveBeenCalledTimes(1); + expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); + const archivePath = path.join("/foo/bar", "cache.tgz"); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] - : ["-cz", "-f", archivePath, "-C", cachePath, "."]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(createTarMock).toHaveBeenCalledTimes(1); + expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(saveCacheMock).toHaveBeenCalledTimes(1); - expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); + expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); @@ -321,32 +350,29 @@ test("save with valid inputs uploads a cache", async () => { const cachePath = path.resolve(inputPath); testUtils.setInput(Inputs.Path, inputPath); - const execMock = jest.spyOn(exec, "exec"); + const cacheId = 4; + const reserveCacheMock = jest + .spyOn(cacheHttpClient, "reserveCache") + .mockImplementationOnce(() => { + return Promise.resolve(cacheId); + }); + + const createTarMock = jest.spyOn(tar, "createTar"); const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); await run(); + expect(reserveCacheMock).toHaveBeenCalledTimes(1); + expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); + const archivePath = path.join("/foo/bar", "cache.tgz"); - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] - : ["-cz", "-f", archivePath, "-C", cachePath, "."]; - - expect(execMock).toHaveBeenCalledTimes(1); - expect(execMock).toHaveBeenCalledWith(`"tar"`, args); + expect(createTarMock).toHaveBeenCalledTimes(1); + expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); expect(saveCacheMock).toHaveBeenCalledTimes(1); - expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); + expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(failedMock).toHaveBeenCalledTimes(0); }); diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts new file mode 100644 index 0000000..55ff4c7 --- /dev/null +++ b/__tests__/tar.test.ts @@ -0,0 +1,58 @@ +import * as exec from "@actions/exec"; +import * as io from "@actions/io"; +import * as tar from "../src/tar"; + +jest.mock("@actions/exec"); +jest.mock("@actions/io"); + +beforeAll(() => { + jest.spyOn(io, "which").mockImplementation(tool => { + return Promise.resolve(tool); + }); +}); + +test("extract tar", async () => { + const mkdirMock = jest.spyOn(io, "mkdirP"); + const execMock = jest.spyOn(exec, "exec"); + + const archivePath = "cache.tar"; + const targetDirectory = "~/.npm/cache"; + await tar.extractTar(archivePath, targetDirectory); + + expect(mkdirMock).toHaveBeenCalledWith(targetDirectory); + + const IS_WINDOWS = process.platform === "win32"; + const tarPath = IS_WINDOWS + ? `${process.env["windir"]}\\System32\\tar.exe` + : "tar"; + expect(execMock).toHaveBeenCalledTimes(1); + expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ + "-xz", + "-f", + archivePath, + "-C", + targetDirectory + ]); +}); + +test("create tar", async () => { + const execMock = jest.spyOn(exec, "exec"); + + const archivePath = "cache.tar"; + const sourceDirectory = "~/.npm/cache"; + await tar.createTar(archivePath, sourceDirectory); + + const IS_WINDOWS = process.platform === "win32"; + const tarPath = IS_WINDOWS + ? `${process.env["windir"]}\\System32\\tar.exe` + : "tar"; + expect(execMock).toHaveBeenCalledTimes(1); + expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ + "-cz", + "-f", + archivePath, + "-C", + sourceDirectory, + "." + ]); +}); diff --git a/dist/restore/index.js b/dist/restore/index.js index f449334..af5fd45 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1496,47 +1496,63 @@ const fs = __importStar(__webpack_require__(747)); const Handlers_1 = __webpack_require__(941); const HttpClient_1 = __webpack_require__(874); const RestClient_1 = __webpack_require__(105); -function getCacheUrl() { +const utils = __importStar(__webpack_require__(443)); +function isSuccessStatusCode(statusCode) { + return statusCode >= 200 && statusCode < 300; +} +function isRetryableStatusCode(statusCode) { + const retryableStatusCodes = [ + HttpClient_1.HttpCodes.BadGateway, + HttpClient_1.HttpCodes.ServiceUnavailable, + HttpClient_1.HttpCodes.GatewayTimeout + ]; + return retryableStatusCodes.includes(statusCode); +} +function getCacheApiUrl() { // Ideally we just use ACTIONS_CACHE_URL - const cacheUrl = (process.env["ACTIONS_CACHE_URL"] || + const baseUrl = (process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_RUNTIME_URL"] || "").replace("pipelines", "artifactcache"); - if (!cacheUrl) { + if (!baseUrl) { throw new Error("Cache Service Url not found, unable to restore cache."); } - core.debug(`Cache Url: ${cacheUrl}`); - return cacheUrl; + core.debug(`Cache Url: ${baseUrl}`); + return `${baseUrl}_apis/artifactcache/`; } function createAcceptHeader(type, apiVersion) { return `${type};api-version=${apiVersion}`; } function getRequestOptions() { const requestOptions = { - acceptHeader: createAcceptHeader("application/json", "5.2-preview.1") + acceptHeader: createAcceptHeader("application/json", "6.0-preview.1") }; return requestOptions; } +function createRestClient() { + const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; + const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); + return new RestClient_1.RestClient("actions/cache", getCacheApiUrl(), [ + bearerCredentialHandler + ]); +} function getCacheEntry(keys) { + var _a; return __awaiter(this, void 0, void 0, function* () { - const cacheUrl = getCacheUrl(); - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent(keys.join(","))}`; - const restClient = new RestClient_1.RestClient("actions/cache", cacheUrl, [ - bearerCredentialHandler - ]); + const restClient = createRestClient(); + const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; const response = yield restClient.get(resource, getRequestOptions()); if (response.statusCode === 204) { return null; } - if (response.statusCode !== 200) { + if (!isSuccessStatusCode(response.statusCode)) { throw new Error(`Cache service responded with ${response.statusCode}`); } const cacheResult = response.result; - if (!cacheResult || !cacheResult.archiveLocation) { + const cacheDownloadUrl = (_a = cacheResult) === null || _a === void 0 ? void 0 : _a.archiveLocation; + if (!cacheDownloadUrl) { throw new Error("Cache not found."); } - core.setSecret(cacheResult.archiveLocation); + core.setSecret(cacheDownloadUrl); core.debug(`Cache Result:`); core.debug(JSON.stringify(cacheResult)); return cacheResult; @@ -1552,34 +1568,117 @@ function pipeResponseToStream(response, stream) { }); }); } -function downloadCache(cacheEntry, archivePath) { +function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient_1.HttpClient("actions/cache"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const downloadResponse = yield httpClient.get(cacheEntry.archiveLocation); + const downloadResponse = yield httpClient.get(archiveLocation); yield pipeResponseToStream(downloadResponse, stream); }); } exports.downloadCache = downloadCache; -function saveCache(key, archivePath) { +// Reserve Cache +function reserveCache(key) { + var _a, _b, _c; return __awaiter(this, void 0, void 0, function* () { - const stream = fs.createReadStream(archivePath); - const cacheUrl = getCacheUrl(); - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`; - const postUrl = cacheUrl + resource; - const restClient = new RestClient_1.RestClient("actions/cache", undefined, [ - bearerCredentialHandler - ]); + const restClient = createRestClient(); + const reserveCacheRequest = { + key + }; + const response = yield restClient.create("caches", reserveCacheRequest, getRequestOptions()); + return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1); + }); +} +exports.reserveCache = reserveCache; +function getContentRange(start, end) { + // Format: `bytes start-end/filesize + // start and end are inclusive + // filesize can be * + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/* + return `bytes ${start}-${end}/*`; +} +function uploadChunk(restClient, resourceUrl, data, start, end) { + return __awaiter(this, void 0, void 0, function* () { + core.debug(`Uploading chunk of size ${end - + start + + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); const requestOptions = getRequestOptions(); requestOptions.additionalHeaders = { - "Content-Type": "application/octet-stream" + "Content-Type": "application/octet-stream", + "Content-Range": getContentRange(start, end) }; - const response = yield restClient.uploadStream("POST", postUrl, stream, requestOptions); - if (response.statusCode !== 200) { - throw new Error(`Cache service responded with ${response.statusCode}`); + const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { + return yield restClient.uploadStream("PATCH", resourceUrl, data, requestOptions); + }); + const response = yield uploadChunkRequest(); + if (isSuccessStatusCode(response.statusCode)) { + return; + } + if (isRetryableStatusCode(response.statusCode)) { + core.debug(`Received ${response.statusCode}, retrying chunk at offset ${start}.`); + const retryResponse = yield uploadChunkRequest(); + if (isSuccessStatusCode(retryResponse.statusCode)) { + return; + } + } + throw new Error(`Cache service responded with ${response.statusCode} during chunk upload.`); + }); +} +function uploadFile(restClient, cacheId, archivePath) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Upload Chunks + const fileSize = fs.statSync(archivePath).size; + const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); + const fd = fs.openSync(archivePath, "r"); + const concurrency = (_a = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]), (_a !== null && _a !== void 0 ? _a : 4)); // # of HTTP requests in parallel + const MAX_CHUNK_SIZE = (_b = Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]), (_b !== null && _b !== void 0 ? _b : 32 * 1024 * 1024)); // 32 MB Chunks + core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); + const parallelUploads = [...new Array(concurrency).keys()]; + core.debug("Awaiting all uploads"); + let offset = 0; + try { + yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + const chunk = fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }); + yield uploadChunk(restClient, resourceUrl, chunk, start, end); + } + }))); + } + finally { + fs.closeSync(fd); + } + return; + }); +} +function commitCache(restClient, cacheId, filesize) { + return __awaiter(this, void 0, void 0, function* () { + const requestOptions = getRequestOptions(); + const commitCacheRequest = { size: filesize }; + return yield restClient.create(`caches/${cacheId.toString()}`, commitCacheRequest, requestOptions); + }); +} +function saveCache(cacheId, archivePath) { + return __awaiter(this, void 0, void 0, function* () { + const restClient = createRestClient(); + core.debug("Upload cache"); + yield uploadFile(restClient, cacheId, archivePath); + // Commit Cache + core.debug("Commiting cache"); + const cacheSize = utils.getArchiveFileSize(archivePath); + const commitCacheResponse = yield commitCache(restClient, cacheId, cacheSize); + if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { + throw new Error(`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`); } core.info("Cache saved successfully"); }); @@ -2991,13 +3090,13 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -const exec_1 = __webpack_require__(986); -const io = __importStar(__webpack_require__(1)); const path = __importStar(__webpack_require__(622)); const cacheHttpClient = __importStar(__webpack_require__(154)); const constants_1 = __webpack_require__(694); +const tar_1 = __webpack_require__(943); const utils = __importStar(__webpack_require__(443)); function run() { + var _a; return __awaiter(this, void 0, void 0, function* () { try { // Validate inputs, this can cause task failure @@ -3035,7 +3134,7 @@ function run() { } try { const cacheEntry = yield cacheHttpClient.getCacheEntry(keys); - if (!cacheEntry) { + if (!((_a = cacheEntry) === null || _a === void 0 ? void 0 : _a.archiveLocation)) { core.info(`Cache not found for input keys: ${keys.join(", ")}.`); return; } @@ -3044,27 +3143,10 @@ function run() { // Store the cache result utils.setCacheState(cacheEntry); // Download the cache from the cache entry - yield cacheHttpClient.downloadCache(cacheEntry, archivePath); + yield cacheHttpClient.downloadCache(cacheEntry.archiveLocation, archivePath); const archiveFileSize = utils.getArchiveFileSize(archivePath); core.info(`Cache Size: ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B)`); - // Create directory to extract tar into - yield io.mkdirP(cachePath); - // http://man7.org/linux/man-pages/man1/tar.1.html - // tar [-options] [files or directories which to add into archive] - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-xz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/") - ] - : ["-xz", "-f", archivePath, "-C", cachePath]; - const tarPath = yield io.which("tar", true); - core.debug(`Tar Path: ${tarPath}`); - yield exec_1.exec(`"${tarPath}"`, args); + yield tar_1.extractTar(archivePath, cachePath); const isExactKeyMatch = utils.isExactKeyMatch(primaryKey, cacheEntry); utils.setCacheHitOutput(isExactKeyMatch); core.info(`Cache restored from key: ${cacheEntry && cacheEntry.cacheKey}`); @@ -5160,6 +5242,79 @@ var personalaccesstoken_1 = __webpack_require__(327); exports.PersonalAccessTokenCredentialHandler = personalaccesstoken_1.PersonalAccessTokenCredentialHandler; +/***/ }), + +/***/ 943: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const exec_1 = __webpack_require__(986); +const io = __importStar(__webpack_require__(1)); +const fs_1 = __webpack_require__(747); +function getTarPath() { + return __awaiter(this, void 0, void 0, function* () { + // Explicitly use BSD Tar on Windows + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; + if (fs_1.existsSync(systemTar)) { + return systemTar; + } + } + return yield io.which("tar", true); + }); +} +function execTar(args) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + try { + yield exec_1.exec(`"${yield getTarPath()}"`, args); + } + catch (error) { + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`); + } + throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`); + } + }); +} +function extractTar(archivePath, targetDirectory) { + return __awaiter(this, void 0, void 0, function* () { + // Create directory to extract tar into + yield io.mkdirP(targetDirectory); + const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + yield execTar(args); + }); +} +exports.extractTar = extractTar; +function createTar(archivePath, sourceDirectory) { + return __awaiter(this, void 0, void 0, function* () { + const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + yield execTar(args); + }); +} +exports.createTar = createTar; + + /***/ }), /***/ 986: diff --git a/dist/save/index.js b/dist/save/index.js index 534e0cf..f46ead5 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1496,47 +1496,63 @@ const fs = __importStar(__webpack_require__(747)); const Handlers_1 = __webpack_require__(941); const HttpClient_1 = __webpack_require__(874); const RestClient_1 = __webpack_require__(105); -function getCacheUrl() { +const utils = __importStar(__webpack_require__(443)); +function isSuccessStatusCode(statusCode) { + return statusCode >= 200 && statusCode < 300; +} +function isRetryableStatusCode(statusCode) { + const retryableStatusCodes = [ + HttpClient_1.HttpCodes.BadGateway, + HttpClient_1.HttpCodes.ServiceUnavailable, + HttpClient_1.HttpCodes.GatewayTimeout + ]; + return retryableStatusCodes.includes(statusCode); +} +function getCacheApiUrl() { // Ideally we just use ACTIONS_CACHE_URL - const cacheUrl = (process.env["ACTIONS_CACHE_URL"] || + const baseUrl = (process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_RUNTIME_URL"] || "").replace("pipelines", "artifactcache"); - if (!cacheUrl) { + if (!baseUrl) { throw new Error("Cache Service Url not found, unable to restore cache."); } - core.debug(`Cache Url: ${cacheUrl}`); - return cacheUrl; + core.debug(`Cache Url: ${baseUrl}`); + return `${baseUrl}_apis/artifactcache/`; } function createAcceptHeader(type, apiVersion) { return `${type};api-version=${apiVersion}`; } function getRequestOptions() { const requestOptions = { - acceptHeader: createAcceptHeader("application/json", "5.2-preview.1") + acceptHeader: createAcceptHeader("application/json", "6.0-preview.1") }; return requestOptions; } +function createRestClient() { + const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; + const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); + return new RestClient_1.RestClient("actions/cache", getCacheApiUrl(), [ + bearerCredentialHandler + ]); +} function getCacheEntry(keys) { + var _a; return __awaiter(this, void 0, void 0, function* () { - const cacheUrl = getCacheUrl(); - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent(keys.join(","))}`; - const restClient = new RestClient_1.RestClient("actions/cache", cacheUrl, [ - bearerCredentialHandler - ]); + const restClient = createRestClient(); + const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; const response = yield restClient.get(resource, getRequestOptions()); if (response.statusCode === 204) { return null; } - if (response.statusCode !== 200) { + if (!isSuccessStatusCode(response.statusCode)) { throw new Error(`Cache service responded with ${response.statusCode}`); } const cacheResult = response.result; - if (!cacheResult || !cacheResult.archiveLocation) { + const cacheDownloadUrl = (_a = cacheResult) === null || _a === void 0 ? void 0 : _a.archiveLocation; + if (!cacheDownloadUrl) { throw new Error("Cache not found."); } - core.setSecret(cacheResult.archiveLocation); + core.setSecret(cacheDownloadUrl); core.debug(`Cache Result:`); core.debug(JSON.stringify(cacheResult)); return cacheResult; @@ -1552,34 +1568,117 @@ function pipeResponseToStream(response, stream) { }); }); } -function downloadCache(cacheEntry, archivePath) { +function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient_1.HttpClient("actions/cache"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const downloadResponse = yield httpClient.get(cacheEntry.archiveLocation); + const downloadResponse = yield httpClient.get(archiveLocation); yield pipeResponseToStream(downloadResponse, stream); }); } exports.downloadCache = downloadCache; -function saveCache(key, archivePath) { +// Reserve Cache +function reserveCache(key) { + var _a, _b, _c; return __awaiter(this, void 0, void 0, function* () { - const stream = fs.createReadStream(archivePath); - const cacheUrl = getCacheUrl(); - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`; - const postUrl = cacheUrl + resource; - const restClient = new RestClient_1.RestClient("actions/cache", undefined, [ - bearerCredentialHandler - ]); + const restClient = createRestClient(); + const reserveCacheRequest = { + key + }; + const response = yield restClient.create("caches", reserveCacheRequest, getRequestOptions()); + return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1); + }); +} +exports.reserveCache = reserveCache; +function getContentRange(start, end) { + // Format: `bytes start-end/filesize + // start and end are inclusive + // filesize can be * + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/* + return `bytes ${start}-${end}/*`; +} +function uploadChunk(restClient, resourceUrl, data, start, end) { + return __awaiter(this, void 0, void 0, function* () { + core.debug(`Uploading chunk of size ${end - + start + + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); const requestOptions = getRequestOptions(); requestOptions.additionalHeaders = { - "Content-Type": "application/octet-stream" + "Content-Type": "application/octet-stream", + "Content-Range": getContentRange(start, end) }; - const response = yield restClient.uploadStream("POST", postUrl, stream, requestOptions); - if (response.statusCode !== 200) { - throw new Error(`Cache service responded with ${response.statusCode}`); + const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { + return yield restClient.uploadStream("PATCH", resourceUrl, data, requestOptions); + }); + const response = yield uploadChunkRequest(); + if (isSuccessStatusCode(response.statusCode)) { + return; + } + if (isRetryableStatusCode(response.statusCode)) { + core.debug(`Received ${response.statusCode}, retrying chunk at offset ${start}.`); + const retryResponse = yield uploadChunkRequest(); + if (isSuccessStatusCode(retryResponse.statusCode)) { + return; + } + } + throw new Error(`Cache service responded with ${response.statusCode} during chunk upload.`); + }); +} +function uploadFile(restClient, cacheId, archivePath) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Upload Chunks + const fileSize = fs.statSync(archivePath).size; + const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); + const fd = fs.openSync(archivePath, "r"); + const concurrency = (_a = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]), (_a !== null && _a !== void 0 ? _a : 4)); // # of HTTP requests in parallel + const MAX_CHUNK_SIZE = (_b = Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]), (_b !== null && _b !== void 0 ? _b : 32 * 1024 * 1024)); // 32 MB Chunks + core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); + const parallelUploads = [...new Array(concurrency).keys()]; + core.debug("Awaiting all uploads"); + let offset = 0; + try { + yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + const chunk = fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }); + yield uploadChunk(restClient, resourceUrl, chunk, start, end); + } + }))); + } + finally { + fs.closeSync(fd); + } + return; + }); +} +function commitCache(restClient, cacheId, filesize) { + return __awaiter(this, void 0, void 0, function* () { + const requestOptions = getRequestOptions(); + const commitCacheRequest = { size: filesize }; + return yield restClient.create(`caches/${cacheId.toString()}`, commitCacheRequest, requestOptions); + }); +} +function saveCache(cacheId, archivePath) { + return __awaiter(this, void 0, void 0, function* () { + const restClient = createRestClient(); + core.debug("Upload cache"); + yield uploadFile(restClient, cacheId, archivePath); + // Commit Cache + core.debug("Commiting cache"); + const cacheSize = utils.getArchiveFileSize(archivePath); + const commitCacheResponse = yield commitCache(restClient, cacheId, cacheSize); + if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { + throw new Error(`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`); } core.info("Cache saved successfully"); }); @@ -2879,11 +2978,10 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -const exec_1 = __webpack_require__(986); -const io = __importStar(__webpack_require__(1)); const path = __importStar(__webpack_require__(622)); const cacheHttpClient = __importStar(__webpack_require__(154)); const constants_1 = __webpack_require__(694); +const tar_1 = __webpack_require__(943); const utils = __importStar(__webpack_require__(443)); function run() { return __awaiter(this, void 0, void 0, function* () { @@ -2905,35 +3003,27 @@ function run() { core.info(`Cache hit occurred on the primary key ${primaryKey}, not saving cache.`); return; } + core.debug("Reserving Cache"); + const cacheId = yield cacheHttpClient.reserveCache(primaryKey); + if (cacheId == -1) { + core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`); + return; + } + core.debug(`Cache ID: ${cacheId}`); const cachePath = utils.resolvePath(core.getInput(constants_1.Inputs.Path, { required: true })); core.debug(`Cache Path: ${cachePath}`); const archivePath = path.join(yield utils.createTempDirectory(), "cache.tgz"); core.debug(`Archive Path: ${archivePath}`); - // http://man7.org/linux/man-pages/man1/tar.1.html - // tar [-options] [files or directories which to add into archive] - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] - : ["-cz", "-f", archivePath, "-C", cachePath, "."]; - const tarPath = yield io.which("tar", true); - core.debug(`Tar Path: ${tarPath}`); - yield exec_1.exec(`"${tarPath}"`, args); - const fileSizeLimit = 400 * 1024 * 1024; // 400MB + yield tar_1.createTar(archivePath, cachePath); + const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit const archiveFileSize = utils.getArchiveFileSize(archivePath); core.debug(`File Size: ${archiveFileSize}`); if (archiveFileSize > fileSizeLimit) { - utils.logWarning(`Cache size of ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.`); + utils.logWarning(`Cache size of ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B) is over the 2GB limit, not saving cache.`); return; } - yield cacheHttpClient.saveCache(primaryKey, archivePath); + core.debug(`Saving Cache (ID: ${cacheId})`); + yield cacheHttpClient.saveCache(cacheId, archivePath); } catch (error) { utils.logWarning(error.message); @@ -5133,6 +5223,79 @@ var personalaccesstoken_1 = __webpack_require__(327); exports.PersonalAccessTokenCredentialHandler = personalaccesstoken_1.PersonalAccessTokenCredentialHandler; +/***/ }), + +/***/ 943: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const exec_1 = __webpack_require__(986); +const io = __importStar(__webpack_require__(1)); +const fs_1 = __webpack_require__(747); +function getTarPath() { + return __awaiter(this, void 0, void 0, function* () { + // Explicitly use BSD Tar on Windows + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; + if (fs_1.existsSync(systemTar)) { + return systemTar; + } + } + return yield io.which("tar", true); + }); +} +function execTar(args) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + try { + yield exec_1.exec(`"${yield getTarPath()}"`, args); + } + catch (error) { + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`); + } + throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`); + } + }); +} +function extractTar(archivePath, targetDirectory) { + return __awaiter(this, void 0, void 0, function* () { + // Create directory to extract tar into + yield io.mkdirP(targetDirectory); + const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + yield execTar(args); + }); +} +exports.extractTar = extractTar; +function createTar(archivePath, sourceDirectory) { + return __awaiter(this, void 0, void 0, function* () { + const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + yield execTar(args); + }); +} +exports.createTar = createTar; + + /***/ }), /***/ 986: diff --git a/examples.md b/examples.md index 7d66b62..948179c 100644 --- a/examples.md +++ b/examples.md @@ -1,27 +1,20 @@ # Examples -- [Examples](#examples) - - [C# - Nuget](#c---nuget) - - [Elixir - Mix](#elixir---mix) - - [Go - Modules](#go---modules) - - [Java - Gradle](#java---gradle) - - [Java - Maven](#java---maven) - - [Node - npm](#node---npm) - - [macOS and Ubuntu](#macos-and-ubuntu) - - [Windows](#windows) - - [Using multiple systems and `npm config`](#using-multiple-systems-and-npm-config) - - [Node - Yarn](#node---yarn) - - [PHP - Composer](#php---composer) - - [Python - pip](#python---pip) - - [Simple example](#simple-example) - - [Multiple OS's in a workflow](#multiple-oss-in-a-workflow) - - [Using a script to get cache location](#using-a-script-to-get-cache-location) - - [Ruby - Gem](#ruby---gem) - - [Rust - Cargo](#rust---cargo) - - [Swift, Objective-C - Carthage](#swift-objective-c---carthage) - - [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods) +- [C# - NuGet](#c---nuget) +- [Elixir - Mix](#elixir---mix) +- [Go - Modules](#go---modules) +- [Java - Gradle](#java---gradle) +- [Java - Maven](#java---maven) +- [Node - npm](#node---npm) +- [Node - Yarn](#node---yarn) +- [PHP - Composer](#php---composer) +- [Python - pip](#python---pip) +- [Ruby - Gem](#ruby---gem) +- [Rust - Cargo](#rust---cargo) +- [Swift, Objective-C - Carthage](#swift-objective-c---carthage) +- [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods) -## C# - Nuget +## C# - NuGet Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/package-references-in-project-files#locking-dependencies): ```yaml @@ -33,6 +26,21 @@ Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/packa ${{ runner.os }}-nuget- ``` +Depending on the environment, huge packages might be pre-installed in the global cache folder. +If you do not want to include them, consider to move the cache folder like below. +>Note: This workflow does not work for projects that require files to be placed in user profile package folder +```yaml +env: + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages +steps: + - uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} + restore-keys: | + ${{ runner.os }}-nuget- +``` + ## Elixir - Mix ```yaml - uses: actions/cache@v1 @@ -226,6 +234,14 @@ Replace `~/.cache/pip` with the correct `path` if not using Ubuntu. restore-keys: | ${{ runner.os }}-gem- ``` +When dependencies are installed later in the workflow, we must specify the same path for the bundler. + +```yaml +- name: Bundle install + run: | + bundle config path vendor/bundle + bundle install --jobs 4 --retry 3 +``` ## Rust - Cargo diff --git a/package-lock.json b/package-lock.json index 2e8413e..d60f8ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cache", - "version": "1.0.3", + "version": "1.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2854,9 +2854,9 @@ "dev": true }, "handlebars": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz", - "integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.3.tgz", + "integrity": "sha512-3yPecJoJHK/4c6aZhSvxOyG4vJKDshV36VHp0iVCDVh7o9w2vwi3NSnL2MMPj3YdduqaBcu7cGbggJQM0br9xA==", "dev": true, "requires": { "neo-async": "^2.6.0", @@ -4859,9 +4859,9 @@ "dev": true }, "prettier": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz", - "integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", "dev": true }, "prettier-linter-helpers": { @@ -5983,15 +5983,15 @@ } }, "typescript": { - "version": "3.6.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.4.tgz", - "integrity": "sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz", + "integrity": "sha512-Mcr/Qk7hXqFBXMN7p7Lusj1ktCBydylfQM/FZCk5glCNQJrCUKPkMHdo9R0MTFWsC/4kPFvDS0fDPvukfCkFsw==", "dev": true }, "uglify-js": { - "version": "3.6.7", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.7.tgz", - "integrity": "sha512-4sXQDzmdnoXiO+xvmTzQsfIiwrjUCSA95rSP4SEd8tDb51W2TiDOlL76Hl+Kw0Ie42PSItCW8/t6pBNCF2R48A==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.3.tgz", + "integrity": "sha512-7tINm46/3puUA4hCkKYo4Xdts+JDaVC9ZPRcG8Xw9R4nhO/gZgUM3TENq8IF4Vatk8qCig4MzP/c8G4u2BkVQg==", "dev": true, "optional": true, "requires": { diff --git a/package.json b/package.json index 42fbdbe..7de321b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cache", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "Cache dependencies and build outputs", "main": "dist/restore/index.js", @@ -46,8 +46,8 @@ "jest": "^24.8.0", "jest-circus": "^24.7.1", "nock": "^11.7.0", - "prettier": "1.18.2", + "prettier": "^1.19.1", "ts-jest": "^24.0.2", - "typescript": "^3.6.4" + "typescript": "^3.7.3" } } diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 8a2014f..89bca63 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -1,26 +1,49 @@ import * as core from "@actions/core"; import * as fs from "fs"; import { BearerCredentialHandler } from "typed-rest-client/Handlers"; -import { HttpClient } from "typed-rest-client/HttpClient"; +import { HttpClient, HttpCodes } from "typed-rest-client/HttpClient"; import { IHttpClientResponse } from "typed-rest-client/Interfaces"; -import { IRequestOptions, RestClient } from "typed-rest-client/RestClient"; -import { ArtifactCacheEntry } from "./contracts"; +import { + IRequestOptions, + RestClient, + IRestResponse +} from "typed-rest-client/RestClient"; +import { + ArtifactCacheEntry, + CommitCacheRequest, + ReserveCacheRequest, + ReserveCacheResponse +} from "./contracts"; +import * as utils from "./utils/actionUtils"; -function getCacheUrl(): string { +function isSuccessStatusCode(statusCode: number): boolean { + return statusCode >= 200 && statusCode < 300; +} + +function isRetryableStatusCode(statusCode: number): boolean { + const retryableStatusCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout + ]; + return retryableStatusCodes.includes(statusCode); +} + +function getCacheApiUrl(): string { // Ideally we just use ACTIONS_CACHE_URL - const cacheUrl: string = ( + const baseUrl: string = ( process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_RUNTIME_URL"] || "" ).replace("pipelines", "artifactcache"); - if (!cacheUrl) { + if (!baseUrl) { throw new Error( "Cache Service Url not found, unable to restore cache." ); } - core.debug(`Cache Url: ${cacheUrl}`); - return cacheUrl; + core.debug(`Cache Url: ${baseUrl}`); + return `${baseUrl}_apis/artifactcache/`; } function createAcceptHeader(type: string, apiVersion: string): string { @@ -29,26 +52,26 @@ function createAcceptHeader(type: string, apiVersion: string): string { function getRequestOptions(): IRequestOptions { const requestOptions: IRequestOptions = { - acceptHeader: createAcceptHeader("application/json", "5.2-preview.1") + acceptHeader: createAcceptHeader("application/json", "6.0-preview.1") }; return requestOptions; } -export async function getCacheEntry( - keys: string[] -): Promise { - const cacheUrl = getCacheUrl(); +function createRestClient(): RestClient { const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const bearerCredentialHandler = new BearerCredentialHandler(token); - const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent( - keys.join(",") - )}`; - - const restClient = new RestClient("actions/cache", cacheUrl, [ + return new RestClient("actions/cache", getCacheApiUrl(), [ bearerCredentialHandler ]); +} + +export async function getCacheEntry( + keys: string[] +): Promise { + const restClient = createRestClient(); + const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; const response = await restClient.get( resource, @@ -57,14 +80,15 @@ export async function getCacheEntry( if (response.statusCode === 204) { return null; } - if (response.statusCode !== 200) { + if (!isSuccessStatusCode(response.statusCode)) { throw new Error(`Cache service responded with ${response.statusCode}`); } const cacheResult = response.result; - if (!cacheResult || !cacheResult.archiveLocation) { + const cacheDownloadUrl = cacheResult?.archiveLocation; + if (!cacheDownloadUrl) { throw new Error("Cache not found."); } - core.setSecret(cacheResult.archiveLocation); + core.setSecret(cacheDownloadUrl); core.debug(`Cache Result:`); core.debug(JSON.stringify(cacheResult)); @@ -83,46 +107,178 @@ async function pipeResponseToStream( } export async function downloadCache( - cacheEntry: ArtifactCacheEntry, + archiveLocation: string, archivePath: string ): Promise { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient("actions/cache"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const downloadResponse = await httpClient.get(cacheEntry.archiveLocation!); + const downloadResponse = await httpClient.get(archiveLocation); await pipeResponseToStream(downloadResponse, stream); } -export async function saveCache( - key: string, - archivePath: string +// Reserve Cache +export async function reserveCache(key: string): Promise { + const restClient = createRestClient(); + + const reserveCacheRequest: ReserveCacheRequest = { + key + }; + const response = await restClient.create( + "caches", + reserveCacheRequest, + getRequestOptions() + ); + + return response?.result?.cacheId ?? -1; +} + +function getContentRange(start: number, end: number): string { + // Format: `bytes start-end/filesize + // start and end are inclusive + // filesize can be * + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/* + return `bytes ${start}-${end}/*`; +} + +async function uploadChunk( + restClient: RestClient, + resourceUrl: string, + data: NodeJS.ReadableStream, + start: number, + end: number ): Promise { - const stream = fs.createReadStream(archivePath); - - const cacheUrl = getCacheUrl(); - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new BearerCredentialHandler(token); - - const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`; - const postUrl = cacheUrl + resource; - - const restClient = new RestClient("actions/cache", undefined, [ - bearerCredentialHandler - ]); - + core.debug( + `Uploading chunk of size ${end - + start + + 1} bytes at offset ${start} with content range: ${getContentRange( + start, + end + )}` + ); const requestOptions = getRequestOptions(); requestOptions.additionalHeaders = { - "Content-Type": "application/octet-stream" + "Content-Type": "application/octet-stream", + "Content-Range": getContentRange(start, end) }; - const response = await restClient.uploadStream( - "POST", - postUrl, - stream, + const uploadChunkRequest = async (): Promise> => { + return await restClient.uploadStream( + "PATCH", + resourceUrl, + data, + requestOptions + ); + }; + + const response = await uploadChunkRequest(); + if (isSuccessStatusCode(response.statusCode)) { + return; + } + + if (isRetryableStatusCode(response.statusCode)) { + core.debug( + `Received ${response.statusCode}, retrying chunk at offset ${start}.` + ); + const retryResponse = await uploadChunkRequest(); + if (isSuccessStatusCode(retryResponse.statusCode)) { + return; + } + } + + throw new Error( + `Cache service responded with ${response.statusCode} during chunk upload.` + ); +} + +async function uploadFile( + restClient: RestClient, + cacheId: number, + archivePath: string +): Promise { + // Upload Chunks + const fileSize = fs.statSync(archivePath).size; + const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); + const fd = fs.openSync(archivePath, "r"); + + const concurrency = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]) ?? 4; // # of HTTP requests in parallel + const MAX_CHUNK_SIZE = + Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]) ?? 32 * 1024 * 1024; // 32 MB Chunks + core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); + + const parallelUploads = [...new Array(concurrency).keys()]; + core.debug("Awaiting all uploads"); + let offset = 0; + + try { + await Promise.all( + parallelUploads.map(async () => { + while (offset < fileSize) { + const chunkSize = Math.min( + fileSize - offset, + MAX_CHUNK_SIZE + ); + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + const chunk = fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }); + + await uploadChunk( + restClient, + resourceUrl, + chunk, + start, + end + ); + } + }) + ); + } finally { + fs.closeSync(fd); + } + return; +} + +async function commitCache( + restClient: RestClient, + cacheId: number, + filesize: number +): Promise> { + const requestOptions = getRequestOptions(); + const commitCacheRequest: CommitCacheRequest = { size: filesize }; + return await restClient.create( + `caches/${cacheId.toString()}`, + commitCacheRequest, requestOptions ); - if (response.statusCode !== 200) { - throw new Error(`Cache service responded with ${response.statusCode}`); +} + +export async function saveCache( + cacheId: number, + archivePath: string +): Promise { + const restClient = createRestClient(); + + core.debug("Upload cache"); + await uploadFile(restClient, cacheId, archivePath); + + // Commit Cache + core.debug("Commiting cache"); + const cacheSize = utils.getArchiveFileSize(archivePath); + const commitCacheResponse = await commitCache( + restClient, + cacheId, + cacheSize + ); + if (!isSuccessStatusCode(commitCacheResponse.statusCode)) { + throw new Error( + `Cache service responded with ${commitCacheResponse.statusCode} during commit cache.` + ); } core.info("Cache saved successfully"); diff --git a/src/contracts.d.ts b/src/contracts.d.ts index 8478b83..269c7d9 100644 --- a/src/contracts.d.ts +++ b/src/contracts.d.ts @@ -4,3 +4,16 @@ export interface ArtifactCacheEntry { creationTime?: string; archiveLocation?: string; } + +export interface CommitCacheRequest { + size: number; +} + +export interface ReserveCacheRequest { + key: string; + version?: string; +} + +export interface ReserveCacheResponse { + cacheId: number; +} diff --git a/src/restore.ts b/src/restore.ts index 15570cd..4911e7e 100644 --- a/src/restore.ts +++ b/src/restore.ts @@ -1,9 +1,8 @@ import * as core from "@actions/core"; -import { exec } from "@actions/exec"; -import * as io from "@actions/io"; import * as path from "path"; import * as cacheHttpClient from "./cacheHttpClient"; import { Events, Inputs, State } from "./constants"; +import { extractTar } from "./tar"; import * as utils from "./utils/actionUtils"; async function run(): Promise { @@ -61,7 +60,7 @@ async function run(): Promise { try { const cacheEntry = await cacheHttpClient.getCacheEntry(keys); - if (!cacheEntry) { + if (!cacheEntry?.archiveLocation) { core.info( `Cache not found for input keys: ${keys.join(", ")}.` ); @@ -78,7 +77,10 @@ async function run(): Promise { utils.setCacheState(cacheEntry); // Download the cache from the cache entry - await cacheHttpClient.downloadCache(cacheEntry, archivePath); + await cacheHttpClient.downloadCache( + cacheEntry.archiveLocation, + archivePath + ); const archiveFileSize = utils.getArchiveFileSize(archivePath); core.info( @@ -87,27 +89,7 @@ async function run(): Promise { )} MB (${archiveFileSize} B)` ); - // Create directory to extract tar into - await io.mkdirP(cachePath); - - // http://man7.org/linux/man-pages/man1/tar.1.html - // tar [-options] [files or directories which to add into archive] - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-xz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/") - ] - : ["-xz", "-f", archivePath, "-C", cachePath]; - - const tarPath = await io.which("tar", true); - core.debug(`Tar Path: ${tarPath}`); - - await exec(`"${tarPath}"`, args); + await extractTar(archivePath, cachePath); const isExactKeyMatch = utils.isExactKeyMatch( primaryKey, diff --git a/src/save.ts b/src/save.ts index 21f32d3..ee64e42 100644 --- a/src/save.ts +++ b/src/save.ts @@ -1,9 +1,8 @@ import * as core from "@actions/core"; -import { exec } from "@actions/exec"; -import * as io from "@actions/io"; import * as path from "path"; import * as cacheHttpClient from "./cacheHttpClient"; import { Events, Inputs, State } from "./constants"; +import { createTar } from "./tar"; import * as utils from "./utils/actionUtils"; async function run(): Promise { @@ -35,6 +34,15 @@ async function run(): Promise { return; } + core.debug("Reserving Cache"); + const cacheId = await cacheHttpClient.reserveCache(primaryKey); + if (cacheId == -1) { + core.info( + `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.` + ); + return; + } + core.debug(`Cache ID: ${cacheId}`); const cachePath = utils.resolvePath( core.getInput(Inputs.Path, { required: true }) ); @@ -46,38 +54,22 @@ async function run(): Promise { ); core.debug(`Archive Path: ${archivePath}`); - // http://man7.org/linux/man-pages/man1/tar.1.html - // tar [-options] [files or directories which to add into archive] - const IS_WINDOWS = process.platform === "win32"; - const args = IS_WINDOWS - ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] - : ["-cz", "-f", archivePath, "-C", cachePath, "."]; + await createTar(archivePath, cachePath); - const tarPath = await io.which("tar", true); - core.debug(`Tar Path: ${tarPath}`); - await exec(`"${tarPath}"`, args); - - const fileSizeLimit = 400 * 1024 * 1024; // 400MB + const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit const archiveFileSize = utils.getArchiveFileSize(archivePath); core.debug(`File Size: ${archiveFileSize}`); if (archiveFileSize > fileSizeLimit) { utils.logWarning( `Cache size of ~${Math.round( archiveFileSize / (1024 * 1024) - )} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.` + )} MB (${archiveFileSize} B) is over the 2GB limit, not saving cache.` ); return; } - await cacheHttpClient.saveCache(primaryKey, archivePath); + core.debug(`Saving Cache (ID: ${cacheId})`); + await cacheHttpClient.saveCache(cacheId, archivePath); } catch (error) { utils.logWarning(error.message); } diff --git a/src/tar.ts b/src/tar.ts new file mode 100644 index 0000000..1f572d1 --- /dev/null +++ b/src/tar.ts @@ -0,0 +1,47 @@ +import { exec } from "@actions/exec"; +import * as io from "@actions/io"; +import { existsSync } from "fs"; + +async function getTarPath(): Promise { + // Explicitly use BSD Tar on Windows + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; + if (existsSync(systemTar)) { + return systemTar; + } + } + return await io.which("tar", true); +} + +async function execTar(args: string[]): Promise { + try { + await exec(`"${await getTarPath()}"`, args); + } catch (error) { + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + throw new Error( + `Tar failed with error: ${error?.message}. Ensure BSD tar is installed and on the PATH.` + ); + } + throw new Error(`Tar failed with error: ${error?.message}`); + } +} + +export async function extractTar( + archivePath: string, + targetDirectory: string +): Promise { + // Create directory to extract tar into + await io.mkdirP(targetDirectory); + const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + await execTar(args); +} + +export async function createTar( + archivePath: string, + sourceDirectory: string +): Promise { + const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + await execTar(args); +}