src: cleanup flakiness in different parts of the action

src/__tests__/blacksmith-builder.test.ts

@@ -46,7 +46,7 @@ describe('startBlacksmithBuilder', () => {
 
     expect(result).toEqual({addr: null, buildId: null, exposeId: ''});
     expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to resolve dockerfile path. Falling back to a local build.');
-    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'));
+    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'), 'starting blacksmith builder');
   });
 
   test('should handle missing dockerfile path with nofallback=true', async () => {
@@ -55,7 +55,7 @@ describe('startBlacksmithBuilder', () => {
 
     await expect(main.startBlacksmithBuilder(mockInputs)).rejects.toThrow('Failed to resolve dockerfile path');
     expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to resolve dockerfile path. Failing the build because nofallback is set.');
-    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'));
+    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to resolve dockerfile path'), 'starting blacksmith builder');
   });
 
   test('should handle error in setupStickyDisk with nofallback=false', async () => {
@@ -67,7 +67,7 @@ describe('startBlacksmithBuilder', () => {
 
     expect(result).toEqual({addr: null, buildId: null, exposeId: ''});
     expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to obtain Blacksmith builder. Falling back to a local build.');
-    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to obtain Blacksmith builder'));
+    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(new Error('Failed to obtain Blacksmith builder'), 'starting blacksmith builder');
   });
 
   test('should handle error in setupStickyDisk with nofallback=true', async () => {
@@ -78,7 +78,7 @@ describe('startBlacksmithBuilder', () => {
 
     await expect(main.startBlacksmithBuilder(mockInputs)).rejects.toThrow(error);
     expect(core.warning).toHaveBeenCalledWith('Error during Blacksmith builder setup: Failed to obtain Blacksmith builder. Failing the build because nofallback is set.');
-    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(error);
+    expect(reporter.reportBuildPushActionFailure).toHaveBeenCalledWith(error, 'starting blacksmith builder');
   });
 
   test('should successfully start buildkitd when setup succeeds', async () => {

src/main.ts

@@ -81,7 +81,7 @@ export async function startBlacksmithBuilder(inputs: context.Inputs): Promise<{a
   } catch (error) {
     // If the builder setup fails for any reason, we check if we should fallback to a local build.
     // If we should not fallback, we rethrow the error and fail the build.
-    await reporter.reportBuildPushActionFailure(error);
+    await reporter.reportBuildPushActionFailure(error, "starting blacksmith builder");
 
     let errorMessage = `Error during Blacksmith builder setup: ${error.message}`;
     if (error.message.includes('buildkitd')) {
@@ -137,14 +137,13 @@ actionsToolkit.run(
       buildId: null as string | null,
       exposeId: '' as string
     };
-    await core.group(`Starting Blacksmith builder`, async () => {
-      builderInfo = await startBlacksmithBuilder(inputs);
-    });
-
     let buildError: Error | undefined;
     let buildDurationSeconds: string | undefined;
     let ref: string | undefined;
     try {
+      await core.group(`Starting Blacksmith builder`, async () => {
+        builderInfo = await startBlacksmithBuilder(inputs);
+      });
       if (builderInfo.addr) {
         await core.group(`Creating a builder instance`, async () => {
           const name = `blacksmith-${Date.now().toString(36)}`;
@@ -315,22 +314,43 @@ actionsToolkit.run(
             refs: ref ? [ref] : []
           });
         }
-        await shutdownBuildkitd();
+        try {
-        core.info('Shutdown buildkitd');
+          const {stdout} = await execAsync('pgrep -f buildkitd');
-        for (let attempt = 1; attempt <= 10; attempt++) {
+          if (stdout.trim()) {
-          try {
+            await shutdownBuildkitd();
-            await execAsync(`sudo umount ${mountPoint}`);
+            core.info('Shutdown buildkitd');
-            core.debug(`${mountPoint} has been unmounted`);
+          }
-            break;
+        } catch (error) {
-          } catch (error) {
+          // No buildkitd process found, nothing to shutdown
-            if (attempt === 10) {
+          core.debug('No buildkitd process found running');
-              throw error;
+        }
+        try {
+          const {stdout: mountOutput} = await execAsync(`mount | grep ${mountPoint}`);
+          if (mountOutput) {
+            for (let attempt = 1; attempt <= 3; attempt++) {
+              try {
+                await execAsync(`sudo umount ${mountPoint}`);
+                core.debug(`${mountPoint} has been unmounted`);
+                break;
+              } catch (error) {
+                if (attempt === 3) {
+                  throw error;
+                }
+                core.warning(`Unmount failed, retrying (${attempt}/3)...`);
+                await new Promise(resolve => setTimeout(resolve, 100));
+              }
             }
-            core.warning(`Unmount failed, retrying (${attempt}/10)...`);
+            core.info('Unmounted device');
-            await new Promise(resolve => setTimeout(resolve, 300));
+          }
+        } catch (error) {
+          // grep returns exit code 1 when no matches are found.
+          if (error.code === 1) {
+            core.debug('No dangling mounts found to clean up');
+          } else {
+            // Only warn for actual errors, not for the expected case where grep finds nothing.
+            core.warning(`Error during cleanup: ${error.message}`);
           }
         }
-        core.info('Unmounted device');
 
         if (builderInfo.addr) {
           if (!buildError) {
@@ -341,7 +361,7 @@ actionsToolkit.run(
         }
       } catch (error) {
         core.warning(`Error during Blacksmith builder shutdown: ${error.message}`);
-        await reporter.reportBuildPushActionFailure(error);
+        await reporter.reportBuildPushActionFailure(error, "shutting down blacksmith builder");
       } finally {
         if (buildError) {
           try {

src/reporter.ts

@@ -43,7 +43,7 @@ export function createBlacksmithAgentClient() {
   return createClient(StickyDiskService, transport);
 }
 
-export async function reportBuildPushActionFailure(error?: Error) {
+export async function reportBuildPushActionFailure(error?: Error, event?: string) {
   const requestOptions = {
     stickydisk_key: process.env.GITHUB_REPO_NAME || '',
     repo_name: process.env.GITHUB_REPO_NAME || '',
@@ -51,7 +51,7 @@ export async function reportBuildPushActionFailure(error?: Error) {
     arch: process.env.BLACKSMITH_ENV?.includes('arm') ? 'arm64' : 'amd64',
     vm_id: process.env.VM_ID || '',
     petname: process.env.PETNAME || '',
-    message: error?.message || ''
+    message: event ? `${event}: ${error?.message || ''}` : error?.message || ''
   };
   
   const client = createBlacksmithAPIClient();
@@ -66,7 +66,7 @@ export async function reportBuildCompleted(exportRes?: ExportRecordResponse, bla
   }
 
   try {
-    const agentClient = await createBlacksmithAgentClient();
+    const agentClient = createBlacksmithAgentClient();
     
     await agentClient.commitStickyDisk({
       exposeId: exposeId || '',
@@ -117,7 +117,7 @@ export async function reportBuildFailed(dockerBuildId: string | null, dockerBuil
   }
 
   try {
-    const blacksmithAgentClient = await createBlacksmithAgentClient();
+    const blacksmithAgentClient = createBlacksmithAgentClient();
     await blacksmithAgentClient.commitStickyDisk({
       exposeId: exposeId || '',
       stickyDiskKey: process.env.GITHUB_REPO_NAME || '',
@@ -166,17 +166,6 @@ export async function reportBuild(dockerfilePath: string) {
   }
 }
 
-export async function get(client: AxiosInstance, url: string, formData: FormData | null, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
-  return await client.get(url, {
-    ...(formData && {data: formData}),
-    headers: {
-      ...client.defaults.headers.common,
-      ...(formData && {'Content-Type': 'multipart/form-data'})
-    },
-    signal: options?.signal
-  });
-}
-
 export async function post(client: AxiosInstance, url: string, formData: FormData | null, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
   return await client.post(url, formData, {
     headers: {

src/setup_builder.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 import * as core from '@actions/core';
-import {exec} from 'child_process';
+import {exec, execSync} from 'child_process';
 import {promisify} from 'util';
 import * as TOML from '@iarna/toml';
 import * as reporter from './reporter';
@@ -177,7 +177,7 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
 
   // Change permissions on the buildkitd socket to allow non-root access
   const startTime = Date.now();
-  const timeout = 10000; // 10 seconds in milliseconds
+  const timeout = 30000; // 30 seconds in milliseconds
 
   while (Date.now() - startTime < timeout) {
     if (fs.existsSync('/run/buildkit/buildkitd.sock')) {
@@ -185,11 +185,39 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
       await execAsync(`sudo chmod 666 /run/buildkit/buildkitd.sock`);
       break;
     }
-    await new Promise(resolve => setTimeout(resolve, 100)); // Poll every 100ms
+    await new Promise(resolve => setTimeout(resolve, 1000)); // Poll every 100ms
   }
 
   if (!fs.existsSync('/run/buildkit/buildkitd.sock')) {
-    throw new Error('buildkitd socket not found after 10s timeout');
+    throw new Error('buildkitd socket not found after 30s timeout');
+  }
+  // Check that buildkit instance is ready by querying workers for up to 30s
+  const startTimeBuildkitReady = Date.now();
+  const timeoutBuildkitReady = 30000; // 30 seconds
+  
+  while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) {
+    try {
+      const {stdout} = await execAsync('sudo buildctl debug workers');
+      const lines = stdout.trim().split('\n');
+      if (lines.length > 1) { // Check if we have output lines beyond the header
+        break;
+      }
+    } catch (error) {
+      core.debug(`Error checking buildkit workers: ${error.message}`);
+    }
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+
+  // Final check after timeout.
+  try {
+    const {stdout} = await execAsync('sudo buildctl debug workers');
+    const lines = stdout.trim().split('\n');
+    if (lines.length <= 1) {
+      throw new Error('buildkit workers not ready after 30s timeout');
+    }
+  } catch (error) {
+    core.warning(`Error checking buildkit workers: ${error.message}`);
+    throw error;
   }
   return buildkitdAddr;
 }
@@ -199,7 +227,7 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
 export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
   try {
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 10000);
+    const timeoutId = setTimeout(() => controller.abort(), 30000);
 
     let buildResponse: {docker_build_id: string} | null = null;
     let exposeId: string = '';