actions
/
build-push-action
mirror of https://github.com/docker/build-push-action.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

src: cleanup flakiness in different parts of the action

pull/1358/head
Aditya Maru 2024-12-16 22:24:30 -05:00
parent 7227817bb9
commit ac42783fa9
6 changed files with 82 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dist/index.js generated vendored
View File

File diff suppressed because one or more lines are too long

2
dist/index.js.map generated vendored
View File

File diff suppressed because one or more lines are too long

8
src/__tests__/blacksmith-builder.test.ts
View File

@ -46,7 +46,7 @@ describe('startBlacksmithBuilder', () => {
expect(result).toEqual({addr: null, buildId: null, exposeId: ''});
expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to resolve dockerfile path. Falling back to a local build.');
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'));
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'), 'starting blacksmith builder');
});
test('should handle missing dockerfile path with nofallback=true', async () => {
@ -55,7 +55,7 @@ describe('startBlacksmithBuilder', () => {
await expect(main.startBlacksmithBuilder(mockInputs)).rejects.toThrow('Failed to resolve dockerfile path');
expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to resolve dockerfile path. Failing the build because nofallback is set.');
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'));
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'), 'starting blacksmith builder');
});
test('should handle error in setupStickyDisk with nofallback=false', async () => {
@ -67,7 +67,7 @@ describe('startBlacksmithBuilder', () => {
expect(result).toEqual({addr: null, buildId: null, exposeId: ''});
expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to obtain Blacksmith builder. Falling back to a local build.');
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to obtain Blacksmith builder'));
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to obtain Blacksmith builder'), 'starting blacksmith builder');
});
test('should handle error in setupStickyDisk with nofallback=true', async () => {
@ -78,7 +78,7 @@ describe('startBlacksmithBuilder', () => {
await expect(main.startBlacksmithBuilder(mockInputs)).rejects.toThrow(error);
expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to obtain Blacksmith builder. Failing the build because nofallback is set.');
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(error);
expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(error, 'starting blacksmith builder');
});
test('should successfully start buildkitd when setup succeeds', async () => {

58
src/main.ts
View File

@ -81,7 +81,7 @@ export async function startBlacksmithBuilder(inputs: context.Inputs): Promise<{a
} catch (error) {
// If the builder setup fails for any reason, we check if we should fallback to a local build.
// If we should not fallback, we rethrow the error and fail the build.
await reporter.reportBuildPushActionFailure(error);
await reporter.reportBuildPushActionFailure(error, "starting blacksmith builder");
let errorMessage = `Error during Blacksmith builder setup: ${error.message}`;
if (error.message.includes('buildkitd')) {
@ -137,14 +137,13 @@ actionsToolkit.run(
buildId: null as string | null,
exposeId: '' as string
};
await core.group(`Starting Blacksmith builder`, async () => {
builderInfo = await startBlacksmithBuilder(inputs);
});
let buildError: Error | undefined;
let buildDurationSeconds: string | undefined;
let ref: string | undefined;
try {
await core.group(`Starting Blacksmith builder`, async () => {
builderInfo = await startBlacksmithBuilder(inputs);
});
if (builderInfo.addr) {
await core.group(`Creating a builder instance`, async () => {
const name = `blacksmith-${Date.now().toString(36)}`;
@ -315,22 +314,43 @@ actionsToolkit.run(
refs: ref ? [ref] : []
});
}
await shutdownBuildkitd();
core.info('Shutdown buildkitd');
for (let attempt = 1; attempt <= 10; attempt++) {
try {
await execAsync(`sudo umount ${mountPoint}`);
core.debug(`${mountPoint} has been unmounted`);
break;
} catch (error) {
if (attempt === 10) {
throw error;
try {
const {stdout} = await execAsync('pgrep -f buildkitd');
if (stdout.trim()) {
await shutdownBuildkitd();
core.info('Shutdown buildkitd');
}
} catch (error) {
// No buildkitd process found, nothing to shutdown
core.debug('No buildkitd process found running');
}
try {
const {stdout: mountOutput} = await execAsync(`mount | grep ${mountPoint}`);
if (mountOutput) {
for (let attempt = 1; attempt <= 3; attempt++) {
try {
await execAsync(`sudo umount ${mountPoint}`);
core.debug(`${mountPoint} has been unmounted`);
break;
} catch (error) {
if (attempt === 3) {
throw error;
}
core.warning(`Unmount failed, retrying (${attempt}/3)...`);
await new Promise(resolve => setTimeout(resolve, 100));
}
}
core.warning(`Unmount failed, retrying (${attempt}/10)...`);
await new Promise(resolve => setTimeout(resolve, 300));
core.info('Unmounted device');
}
} catch (error) {
// grep returns exit code 1 when no matches are found.
if (error.code === 1) {
core.debug('No dangling mounts found to clean up');
} else {
// Only warn for actual errors, not for the expected case where grep finds nothing.
core.warning(`Error during cleanup: ${error.message}`);
}
}
core.info('Unmounted device');
if (builderInfo.addr) {
if (!buildError) {
@ -341,7 +361,7 @@ actionsToolkit.run(
}
} catch (error) {
core.warning(`Error during Blacksmith builder shutdown: ${error.message}`);
await reporter.reportBuildPushActionFailure(error);
await reporter.reportBuildPushActionFailure(error, "shutting down blacksmith builder");
} finally {
if (buildError) {
try {

19
src/reporter.ts
View File

@ -43,7 +43,7 @@ export function createBlacksmithAgentClient() {
return createClient(StickyDiskService, transport);
}
export async function reportBuildPushActionFailure(error?: Error) {
export async function reportBuildPushActionFailure(error?: Error, event?: string) {
const requestOptions = {
stickydisk_key: process.env.GITHUB_REPO_NAME || '',
repo_name: process.env.GITHUB_REPO_NAME || '',
@ -51,7 +51,7 @@ export async function reportBuildPushActionFailure(error?: Error) {
arch: process.env.BLACKSMITH_ENV?.includes('arm') ? 'arm64' : 'amd64',
vm_id: process.env.VM_ID || '',
petname: process.env.PETNAME || '',
message: error?.message || ''
message: event ? `${event}: ${error?.message || ''}` : error?.message || ''
};
const client = createBlacksmithAPIClient();
@ -66,7 +66,7 @@ export async function reportBuildCompleted(exportRes?: ExportRecordResponse, bla
}
try {
const agentClient = await createBlacksmithAgentClient();
const agentClient = createBlacksmithAgentClient();
await agentClient.commitStickyDisk({
exposeId: exposeId || '',
@ -117,7 +117,7 @@ export async function reportBuildFailed(dockerBuildId: string | null, dockerBuil
}
try {
const blacksmithAgentClient = await createBlacksmithAgentClient();
const blacksmithAgentClient = createBlacksmithAgentClient();
await blacksmithAgentClient.commitStickyDisk({
exposeId: exposeId || '',
stickyDiskKey: process.env.GITHUB_REPO_NAME || '',
@ -166,17 +166,6 @@ export async function reportBuild(dockerfilePath: string) {
}
}
export async function get(client: AxiosInstance, url: string, formData: FormData | null, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
return await client.get(url, {
...(formData && {data: formData}),
headers: {
...client.defaults.headers.common,
...(formData && {'Content-Type': 'multipart/form-data'})
},
signal: options?.signal
});
}
export async function post(client: AxiosInstance, url: string, formData: FormData | null, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
return await client.post(url, formData, {
headers: {

38
src/setup_builder.ts
View File

@ -1,6 +1,6 @@
import * as fs from 'fs';
import * as core from '@actions/core';
import {exec} from 'child_process';
import {exec, execSync} from 'child_process';
import {promisify} from 'util';
import * as TOML from '@iarna/toml';
import * as reporter from './reporter';
@ -177,7 +177,7 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
// Change permissions on the buildkitd socket to allow non-root access
const startTime = Date.now();
const timeout = 10000; // 10 seconds in milliseconds
const timeout = 30000; // 30 seconds in milliseconds
while (Date.now() - startTime < timeout) {
if (fs.existsSync('/run/buildkit/buildkitd.sock')) {
@ -185,11 +185,39 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
await execAsync(`sudo chmod 666 /run/buildkit/buildkitd.sock`);
break;
}
await new Promise(resolve => setTimeout(resolve, 100)); // Poll every 100ms
await new Promise(resolve => setTimeout(resolve, 1000)); // Poll every 100ms
}
if (!fs.existsSync('/run/buildkit/buildkitd.sock')) {
throw new Error('buildkitd socket not found after 10s timeout');
throw new Error('buildkitd socket not found after 30s timeout');
}
// Check that buildkit instance is ready by querying workers for up to 30s
const startTimeBuildkitReady = Date.now();
const timeoutBuildkitReady = 30000; // 30 seconds
while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) {
try {
const {stdout} = await execAsync('sudo buildctl debug workers');
const lines = stdout.trim().split('\n');
if (lines.length > 1) { // Check if we have output lines beyond the header
break;
}
} catch (error) {
core.debug(`Error checking buildkit workers: ${error.message}`);
}
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Final check after timeout.
try {
const {stdout} = await execAsync('sudo buildctl debug workers');
const lines = stdout.trim().split('\n');
if (lines.length <= 1) {
throw new Error('buildkit workers not ready after 30s timeout');
}
} catch (error) {
core.warning(`Error checking buildkit workers: ${error.message}`);
throw error;
}
return buildkitdAddr;
}
@ -199,7 +227,7 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10000);
const timeoutId = setTimeout(() => controller.abort(), 30000);
let buildResponse: {docker_build_id: string} | null = null;
let exposeId: string = '';