diff --git a/.github/workflows/check-dist.yml b/.github/workflows/check-dist.yml deleted file mode 100644 index 62d4a32..0000000 --- a/.github/workflows/check-dist.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Check dist/ - -on: - push: - branches: - - main - paths-ignore: - - '**.md' - pull_request: - paths-ignore: - - '**.md' - workflow_dispatch: - -jobs: - call-check-dist: - name: Check dist/ - uses: actions/reusable-workflows/.github/workflows/check-dist.yml@main - with: - node-version: "20.x" diff --git a/.github/workflows/close-inactive-issues.yml b/.github/workflows/close-inactive-issues.yml deleted file mode 100644 index aea77e9..0000000 --- a/.github/workflows/close-inactive-issues.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Close inactive issues -on: - schedule: - - cron: "30 8 * * *" - -jobs: - close-issues: - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - steps: - - uses: actions/stale@v3 - with: - days-before-issue-stale: 200 - days-before-issue-close: 5 - stale-issue-label: "stale" - stale-issue-message: "This issue is stale because it has been open for 200 days with no activity. Leave a comment to avoid closing this issue in 5 days." - close-issue-message: "This issue was closed because it has been inactive for 5 days since being marked as stale." - days-before-pr-stale: -1 - days-before-pr-close: -1 - repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index f699c5f..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: "Code scanning - action" - -on: - push: - pull_request: - schedule: - - cron: '0 19 * * 0' - -jobs: - CodeQL-Build: - # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest - runs-on: ubuntu-latest - - permissions: - # required for all workflows - security-events: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - # Override language selection by uncommenting this and choosing your languages - # with: - # languages: go, javascript, csharp, python, cpp, java, ruby - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below). - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/issue-opened-workflow.yml b/.github/workflows/issue-opened-workflow.yml deleted file mode 100644 index 185eb1d..0000000 --- a/.github/workflows/issue-opened-workflow.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Assign issue -on: - issues: - types: [opened] -jobs: - run-action: - runs-on: ubuntu-latest - steps: - - name: Get current oncall - id: oncall - run: | - echo "CURRENT=$(curl --request GET 'https://api.pagerduty.com/oncalls?include[]=users&schedule_ids[]=P5VG2BX&earliest=true' --header 'Authorization: Token token=${{ secrets.PAGERDUTY_TOKEN }}' --header 'Accept: application/vnd.pagerduty+json;version=2' --header 'Content-Type: application/json' | jq -r '.oncalls[].user.name')" >> $GITHUB_OUTPUT - - - name: add_assignees - run: | - curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN}}" https://api.github.com/repos/${{github.repository}}/issues/${{ github.event.issue.number}}/assignees -d '{"assignees":["${{steps.oncall.outputs.CURRENT}}"]}' diff --git a/.github/workflows/licensed.yml b/.github/workflows/licensed.yml deleted file mode 100644 index 37f1560..0000000 --- a/.github/workflows/licensed.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Licensed - -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - call-licensed: - name: Licensed - uses: actions/reusable-workflows/.github/workflows/licensed.yml@main diff --git a/.github/workflows/pr-opened-workflow.yml b/.github/workflows/pr-opened-workflow.yml deleted file mode 100644 index 3346d9e..0000000 --- a/.github/workflows/pr-opened-workflow.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Add Reviewer PR -on: - pull_request_target: - types: [opened] -jobs: - run-action: - runs-on: ubuntu-latest - steps: - - name: Get current oncall - id: oncall - run: | - echo "CURRENT=$(curl --request GET 'https://api.pagerduty.com/oncalls?include[]=users&schedule_ids[]=P5VG2BX&earliest=true' --header 'Authorization: Token token=${{ secrets.PAGERDUTY_TOKEN }}' --header 'Accept: application/vnd.pagerduty+json;version=2' --header 'Content-Type: application/json' | jq -r '.oncalls[].user.name')" >> $GITHUB_OUTPUT - - - name: Request Review - run: | - curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN}}" https://api.github.com/repos/${{github.repository}}/pulls/${{ github.event.pull_request.number}}/requested_reviewers -d '{"reviewers":["${{steps.oncall.outputs.CURRENT}}"]}' - - - name: Add Assignee - run: | - curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN}}" https://api.github.com/repos/${{github.repository}}/issues/${{ github.event.pull_request.number}}/assignees -d '{"assignees":["${{steps.oncall.outputs.CURRENT}}"]}' diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 2a53c19..4cc6463 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -9,51 +9,39 @@ on: branches: - main - releases/** + - fix/** + - v4 jobs: - # Build and unit test - build: - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Node.js 20.x - uses: actions/setup-node@v3 - with: - node-version: 20.x - cache: npm - - run: npm ci - - name: Prettier Format Check - run: npm run format-check - - name: ESLint Check - run: npm run lint - - name: Build & Test - run: npm run test - # End to end save and restore test-save: + runs-on: runs-on,env=dev,image=ubuntu22-full-arm64 strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + part_size: [32] + queue_size: [4, 8] fail-fast: false - runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v3 - name: Generate files in working directory shell: bash - run: __tests__/create-cache-files.sh ${{ runner.os }} test-cache + run: | + __tests__/create-cache-files.sh ${{ runner.os }} test-cache + # 5GB + # curl -o test-cache/ubuntu.iso https://releases.ubuntu.com/22.04.3/ubuntu-22.04.3-desktop-amd64.iso + # 2Gb + curl -o test-cache/ubuntu.iso https://releases.ubuntu.com/jammy/ubuntu-22.04.3-live-server-amd64.iso - name: Generate files outside working directory shell: bash run: __tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache - name: Save cache uses: ./ + env: + UPLOAD_PART_SIZE: ${{ matrix.part_size }} + UPLOAD_QUEUE_SIZE: ${{ matrix.queue_size }} with: - key: test-${{ runner.os }}-${{ github.run_id }} + key: test-${{ runner.os }}-${{ github.run_id }}-${{ matrix.part_size }}-${{ matrix.queue_size }} path: | test-cache ~/test-cache @@ -61,16 +49,23 @@ jobs: needs: test-save strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + part_size: [8, 16] + queue_size: [8, 12] fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: runs-on,env=dev,image=ubuntu22-full-arm64 steps: - name: Checkout uses: actions/checkout@v3 - name: Restore cache uses: ./ + env: + DOWNLOAD_PART_SIZE: ${{ matrix.part_size }} + DOWNLOAD_QUEUE_SIZE: ${{ matrix.queue_size }} with: - key: test-${{ runner.os }}-${{ github.run_id }} + key: test-${{ runner.os }}-${{ github.run_id }}-${{ matrix.part_size }}-${{ matrix.queue_size }} + restore-keys: | + test-${{ runner.os }}-${{ github.run_id }}-${{ matrix.part_size }}- + test-${{ runner.os }}-${{ github.run_id }}- path: | test-cache ~/test-cache @@ -80,50 +75,3 @@ jobs: - name: Verify cache files outside working directory shell: bash run: __tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache - - # End to end with proxy - test-proxy-save: - runs-on: ubuntu-latest - container: - image: ubuntu:latest - options: --dns 127.0.0.1 - services: - squid-proxy: - image: ubuntu/squid:latest - ports: - - 3128:3128 - env: - https_proxy: http://squid-proxy:3128 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Generate files - run: __tests__/create-cache-files.sh proxy test-cache - - name: Save cache - uses: ./ - with: - key: test-proxy-${{ github.run_id }} - path: test-cache - test-proxy-restore: - needs: test-proxy-save - runs-on: ubuntu-latest - container: - image: ubuntu:latest - options: --dns 127.0.0.1 - services: - squid-proxy: - image: ubuntu/squid:latest - ports: - - 3128:3128 - env: - https_proxy: http://squid-proxy:3128 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Restore cache - uses: ./ - with: - key: test-proxy-${{ github.run_id }} - path: test-cache - - name: Verify cache - run: __tests__/verify-cache-files.sh proxy test-cache diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js index 71864c9..6ae1d47 100644 --- a/dist/restore-only/index.js +++ b/dist/restore-only/index.js @@ -93769,7 +93769,6 @@ const client_s3_1 = __nccwpck_require__(9250); const { getSignedUrl } = __nccwpck_require__(5052); const fs_1 = __nccwpck_require__(7147); const crypto = __importStar(__nccwpck_require__(6113)); -const options_1 = __nccwpck_require__(6215); const core = __importStar(__nccwpck_require__(2186)); const utils = __importStar(__nccwpck_require__(1518)); const lib_storage_1 = __nccwpck_require__(3087); @@ -93779,6 +93778,10 @@ const bucketName = process.env.RUNS_ON_S3_BUCKET_CACHE; const region = process.env.RUNS_ON_AWS_REGION || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; +const uploadQueueSize = Number(process.env.UPLOAD_QUEUE_SIZE || "4"); +const uploadPartSize = Number(process.env.UPLOAD_PART_SIZE || "32") * 1024 * 1024; +const downloadQueueSize = Number(process.env.DOWNLOAD_QUEUE_SIZE || "8"); +const downloadPartSize = Number(process.env.DOWNLOAD_PART_SIZE || "16") * 1024 * 1024; function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { // don't pass changes upstream const components = paths.slice(); @@ -93855,8 +93858,7 @@ function downloadCache(archiveLocation, archivePath, options) { const url = yield getSignedUrl(s3Client, command, { expiresIn: 3600 }); - const downloadOptions = (0, options_1.getDownloadOptions)(Object.assign(Object.assign({}, options), { downloadConcurrency: 14, concurrentBlobDownloads: true })); - yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, downloadOptions); + yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, Object.assign(Object.assign({}, options), { downloadConcurrency: downloadQueueSize, concurrentBlobDownloads: true, partSize: downloadPartSize })); }); } exports.downloadCache = downloadCache; @@ -93882,16 +93884,17 @@ function saveCache(key, paths, archivePath, { compressionMethod, enableCrossOsAr Body: (0, fs_1.createReadStream)(archivePath) }, // Part size in bytes - partSize: 32 * 1024 * 1024, + partSize: uploadPartSize, // Max concurrency - queueSize: 14 + queueSize: uploadQueueSize }); // Commit Cache const cacheSize = utils.getArchiveFileSizeInBytes(archivePath); core.info(`Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)`); + const totalParts = Math.ceil(cacheSize / uploadPartSize); core.info(`Uploading cache from ${archivePath} to ${bucketName}/${s3Key}`); multipartUpload.on("httpUploadProgress", progress => { - core.info(`Uploaded ${progress.part}/${progress.total}.`); + core.info(`Uploaded part ${progress.part}/${totalParts}.`); }); yield multipartUpload.done(); core.info(`Cache saved successfully.`); @@ -94308,7 +94311,7 @@ function downloadCacheHttpClientConcurrent(archiveLocation, archivePath, options throw new Error(`Could not interpret Content-Length: ${length}`); } const downloads = []; - const blockSize = 32 * 1024 * 1024; + const blockSize = options.partSize; for (let offset = 0; offset < length; offset += blockSize) { const count = Math.min(blockSize, length - offset); downloads.push({ diff --git a/dist/restore/index.js b/dist/restore/index.js index 6159277..c2418a4 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -93769,7 +93769,6 @@ const client_s3_1 = __nccwpck_require__(9250); const { getSignedUrl } = __nccwpck_require__(5052); const fs_1 = __nccwpck_require__(7147); const crypto = __importStar(__nccwpck_require__(6113)); -const options_1 = __nccwpck_require__(6215); const core = __importStar(__nccwpck_require__(2186)); const utils = __importStar(__nccwpck_require__(1518)); const lib_storage_1 = __nccwpck_require__(3087); @@ -93779,6 +93778,10 @@ const bucketName = process.env.RUNS_ON_S3_BUCKET_CACHE; const region = process.env.RUNS_ON_AWS_REGION || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; +const uploadQueueSize = Number(process.env.UPLOAD_QUEUE_SIZE || "4"); +const uploadPartSize = Number(process.env.UPLOAD_PART_SIZE || "32") * 1024 * 1024; +const downloadQueueSize = Number(process.env.DOWNLOAD_QUEUE_SIZE || "8"); +const downloadPartSize = Number(process.env.DOWNLOAD_PART_SIZE || "16") * 1024 * 1024; function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { // don't pass changes upstream const components = paths.slice(); @@ -93855,8 +93858,7 @@ function downloadCache(archiveLocation, archivePath, options) { const url = yield getSignedUrl(s3Client, command, { expiresIn: 3600 }); - const downloadOptions = (0, options_1.getDownloadOptions)(Object.assign(Object.assign({}, options), { downloadConcurrency: 14, concurrentBlobDownloads: true })); - yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, downloadOptions); + yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, Object.assign(Object.assign({}, options), { downloadConcurrency: downloadQueueSize, concurrentBlobDownloads: true, partSize: downloadPartSize })); }); } exports.downloadCache = downloadCache; @@ -93882,16 +93884,17 @@ function saveCache(key, paths, archivePath, { compressionMethod, enableCrossOsAr Body: (0, fs_1.createReadStream)(archivePath) }, // Part size in bytes - partSize: 32 * 1024 * 1024, + partSize: uploadPartSize, // Max concurrency - queueSize: 14 + queueSize: uploadQueueSize }); // Commit Cache const cacheSize = utils.getArchiveFileSizeInBytes(archivePath); core.info(`Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)`); + const totalParts = Math.ceil(cacheSize / uploadPartSize); core.info(`Uploading cache from ${archivePath} to ${bucketName}/${s3Key}`); multipartUpload.on("httpUploadProgress", progress => { - core.info(`Uploaded ${progress.part}/${progress.total}.`); + core.info(`Uploaded part ${progress.part}/${totalParts}.`); }); yield multipartUpload.done(); core.info(`Cache saved successfully.`); @@ -94308,7 +94311,7 @@ function downloadCacheHttpClientConcurrent(archiveLocation, archivePath, options throw new Error(`Could not interpret Content-Length: ${length}`); } const downloads = []; - const blockSize = 32 * 1024 * 1024; + const blockSize = options.partSize; for (let offset = 0; offset < length; offset += blockSize) { const count = Math.min(blockSize, length - offset); downloads.push({ diff --git a/dist/save-only/index.js b/dist/save-only/index.js index fe20579..88822b9 100644 --- a/dist/save-only/index.js +++ b/dist/save-only/index.js @@ -93769,7 +93769,6 @@ const client_s3_1 = __nccwpck_require__(9250); const { getSignedUrl } = __nccwpck_require__(5052); const fs_1 = __nccwpck_require__(7147); const crypto = __importStar(__nccwpck_require__(6113)); -const options_1 = __nccwpck_require__(6215); const core = __importStar(__nccwpck_require__(2186)); const utils = __importStar(__nccwpck_require__(1518)); const lib_storage_1 = __nccwpck_require__(3087); @@ -93779,6 +93778,10 @@ const bucketName = process.env.RUNS_ON_S3_BUCKET_CACHE; const region = process.env.RUNS_ON_AWS_REGION || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; +const uploadQueueSize = Number(process.env.UPLOAD_QUEUE_SIZE || "4"); +const uploadPartSize = Number(process.env.UPLOAD_PART_SIZE || "32") * 1024 * 1024; +const downloadQueueSize = Number(process.env.DOWNLOAD_QUEUE_SIZE || "8"); +const downloadPartSize = Number(process.env.DOWNLOAD_PART_SIZE || "16") * 1024 * 1024; function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { // don't pass changes upstream const components = paths.slice(); @@ -93855,8 +93858,7 @@ function downloadCache(archiveLocation, archivePath, options) { const url = yield getSignedUrl(s3Client, command, { expiresIn: 3600 }); - const downloadOptions = (0, options_1.getDownloadOptions)(Object.assign(Object.assign({}, options), { downloadConcurrency: 14, concurrentBlobDownloads: true })); - yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, downloadOptions); + yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, Object.assign(Object.assign({}, options), { downloadConcurrency: downloadQueueSize, concurrentBlobDownloads: true, partSize: downloadPartSize })); }); } exports.downloadCache = downloadCache; @@ -93882,16 +93884,17 @@ function saveCache(key, paths, archivePath, { compressionMethod, enableCrossOsAr Body: (0, fs_1.createReadStream)(archivePath) }, // Part size in bytes - partSize: 32 * 1024 * 1024, + partSize: uploadPartSize, // Max concurrency - queueSize: 14 + queueSize: uploadQueueSize }); // Commit Cache const cacheSize = utils.getArchiveFileSizeInBytes(archivePath); core.info(`Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)`); + const totalParts = Math.ceil(cacheSize / uploadPartSize); core.info(`Uploading cache from ${archivePath} to ${bucketName}/${s3Key}`); multipartUpload.on("httpUploadProgress", progress => { - core.info(`Uploaded ${progress.part}/${progress.total}.`); + core.info(`Uploaded part ${progress.part}/${totalParts}.`); }); yield multipartUpload.done(); core.info(`Cache saved successfully.`); @@ -94308,7 +94311,7 @@ function downloadCacheHttpClientConcurrent(archiveLocation, archivePath, options throw new Error(`Could not interpret Content-Length: ${length}`); } const downloads = []; - const blockSize = 32 * 1024 * 1024; + const blockSize = options.partSize; for (let offset = 0; offset < length; offset += blockSize) { const count = Math.min(blockSize, length - offset); downloads.push({ diff --git a/dist/save/index.js b/dist/save/index.js index fafa8ca..695d66f 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -93769,7 +93769,6 @@ const client_s3_1 = __nccwpck_require__(9250); const { getSignedUrl } = __nccwpck_require__(5052); const fs_1 = __nccwpck_require__(7147); const crypto = __importStar(__nccwpck_require__(6113)); -const options_1 = __nccwpck_require__(6215); const core = __importStar(__nccwpck_require__(2186)); const utils = __importStar(__nccwpck_require__(1518)); const lib_storage_1 = __nccwpck_require__(3087); @@ -93779,6 +93778,10 @@ const bucketName = process.env.RUNS_ON_S3_BUCKET_CACHE; const region = process.env.RUNS_ON_AWS_REGION || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; +const uploadQueueSize = Number(process.env.UPLOAD_QUEUE_SIZE || "4"); +const uploadPartSize = Number(process.env.UPLOAD_PART_SIZE || "32") * 1024 * 1024; +const downloadQueueSize = Number(process.env.DOWNLOAD_QUEUE_SIZE || "8"); +const downloadPartSize = Number(process.env.DOWNLOAD_PART_SIZE || "16") * 1024 * 1024; function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { // don't pass changes upstream const components = paths.slice(); @@ -93855,8 +93858,7 @@ function downloadCache(archiveLocation, archivePath, options) { const url = yield getSignedUrl(s3Client, command, { expiresIn: 3600 }); - const downloadOptions = (0, options_1.getDownloadOptions)(Object.assign(Object.assign({}, options), { downloadConcurrency: 14, concurrentBlobDownloads: true })); - yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, downloadOptions); + yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(url, archivePath, Object.assign(Object.assign({}, options), { downloadConcurrency: downloadQueueSize, concurrentBlobDownloads: true, partSize: downloadPartSize })); }); } exports.downloadCache = downloadCache; @@ -93882,16 +93884,17 @@ function saveCache(key, paths, archivePath, { compressionMethod, enableCrossOsAr Body: (0, fs_1.createReadStream)(archivePath) }, // Part size in bytes - partSize: 32 * 1024 * 1024, + partSize: uploadPartSize, // Max concurrency - queueSize: 14 + queueSize: uploadQueueSize }); // Commit Cache const cacheSize = utils.getArchiveFileSizeInBytes(archivePath); core.info(`Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)`); + const totalParts = Math.ceil(cacheSize / uploadPartSize); core.info(`Uploading cache from ${archivePath} to ${bucketName}/${s3Key}`); multipartUpload.on("httpUploadProgress", progress => { - core.info(`Uploaded ${progress.part}/${progress.total}.`); + core.info(`Uploaded part ${progress.part}/${totalParts}.`); }); yield multipartUpload.done(); core.info(`Cache saved successfully.`); @@ -94308,7 +94311,7 @@ function downloadCacheHttpClientConcurrent(archiveLocation, archivePath, options throw new Error(`Could not interpret Content-Length: ${length}`); } const downloads = []; - const blockSize = 32 * 1024 * 1024; + const blockSize = options.partSize; for (let offset = 0; offset < length; offset += blockSize) { const count = Math.min(blockSize, length - offset); downloads.push({ diff --git a/src/custom/backend.ts b/src/custom/backend.ts index 3e92333..d872afa 100644 --- a/src/custom/backend.ts +++ b/src/custom/backend.ts @@ -31,6 +31,13 @@ const region = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; +const uploadQueueSize = Number(process.env.UPLOAD_QUEUE_SIZE || "4"); +const uploadPartSize = + Number(process.env.UPLOAD_PART_SIZE || "32") * 1024 * 1024; +const downloadQueueSize = Number(process.env.DOWNLOAD_QUEUE_SIZE || "8"); +const downloadPartSize = + Number(process.env.DOWNLOAD_PART_SIZE || "16") * 1024 * 1024; + export function getCacheVersion( paths: string[], compressionMethod?: CompressionMethod, @@ -140,12 +147,12 @@ export async function downloadCache( const url = await getSignedUrl(s3Client, command, { expiresIn: 3600 }); - const downloadOptions = getDownloadOptions({ + await downloadCacheHttpClientConcurrent(url, archivePath, { ...options, - downloadConcurrency: 14, - concurrentBlobDownloads: true + downloadConcurrency: downloadQueueSize, + concurrentBlobDownloads: true, + partSize: downloadPartSize }); - await downloadCacheHttpClientConcurrent(url, archivePath, downloadOptions); } export async function saveCache( @@ -176,12 +183,10 @@ export async function saveCache( Key: s3Key, Body: createReadStream(archivePath) }, - // Part size in bytes - partSize: 32 * 1024 * 1024, - + partSize: uploadPartSize, // Max concurrency - queueSize: 14 + queueSize: uploadQueueSize }); // Commit Cache @@ -192,9 +197,10 @@ export async function saveCache( )} MB (${cacheSize} B)` ); + const totalParts = Math.ceil(cacheSize / uploadPartSize); core.info(`Uploading cache from ${archivePath} to ${bucketName}/${s3Key}`); multipartUpload.on("httpUploadProgress", progress => { - core.info(`Uploaded ${progress.part}/${progress.total}.`); + core.info(`Uploaded part ${progress.part}/${totalParts}.`); }); await multipartUpload.done(); diff --git a/src/custom/downloadUtils.ts b/src/custom/downloadUtils.ts index 32c77b8..5e311a0 100644 --- a/src/custom/downloadUtils.ts +++ b/src/custom/downloadUtils.ts @@ -6,6 +6,10 @@ import * as fs from "fs"; import { DownloadOptions } from "@actions/cache/lib/options"; import { retryHttpClientResponse } from "@actions/cache/lib/internal/requestUtils"; +export interface RunsOnDownloadOptions extends DownloadOptions { + partSize: number; +} + /** * Class for tracking the download state and displaying stats. */ @@ -149,7 +153,7 @@ export class DownloadProgress { export async function downloadCacheHttpClientConcurrent( archiveLocation: string, archivePath: fs.PathLike, - options: DownloadOptions + options: RunsOnDownloadOptions ): Promise { const archiveDescriptor = await fs.promises.open(archivePath, "w"); const httpClient = new HttpClient("actions/cache", undefined, { @@ -185,7 +189,7 @@ export async function downloadCacheHttpClientConcurrent( promiseGetter: () => Promise; }[] = []; - const blockSize = 32 * 1024 * 1024; + const blockSize = options.partSize; for (let offset = 0; offset < length; offset += blockSize) { const count = Math.min(blockSize, length - offset);