feedback improvements
parent
e6686e4d61
commit
cb6b30300e
|
|
@ -3281,13 +3281,14 @@ class TaskService {
|
||||||
if (taskElement === undefined) {
|
if (taskElement === undefined) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
taskElement.overrides = {};
|
const extendedTask = taskElement;
|
||||||
taskElement.attachments = [];
|
extendedTask.overrides = {};
|
||||||
if (taskElement.createdAt === undefined) {
|
extendedTask.attachments = [];
|
||||||
cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
if (extendedTask.createdAt === undefined) {
|
||||||
|
cloud_runner_logger_1.default.log(`Skipping ${extendedTask.taskDefinitionArn} no createdAt date`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
result.push({ taskElement, element });
|
result.push({ taskElement: extendedTask, element });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -4421,12 +4422,33 @@ class LocalCloudRunner {
|
||||||
cloud_runner_logger_1.default.log(commands);
|
cloud_runner_logger_1.default.log(commands);
|
||||||
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
||||||
if (process.platform === 'win32') {
|
if (process.platform === 'win32') {
|
||||||
const inline = commands
|
// Properly escape the command string for embedding in a double-quoted bash string.
|
||||||
.replace(/"/g, '\\"')
|
// Order matters: backslashes must be escaped first to avoid double-escaping.
|
||||||
|
const escapeForBashDoubleQuotes = (stringValue) => {
|
||||||
|
return stringValue
|
||||||
|
.replace(/\\/g, '\\\\') // Escape backslashes first
|
||||||
|
.replace(/\$/g, '\\$') // Escape dollar signs to prevent variable expansion
|
||||||
|
.replace(/`/g, '\\`') // Escape backticks to prevent command substitution
|
||||||
|
.replace(/"/g, '\\"'); // Escape double quotes
|
||||||
|
};
|
||||||
|
// Split commands by newlines and escape each line
|
||||||
|
const lines = commands
|
||||||
.replace(/\r/g, '')
|
.replace(/\r/g, '')
|
||||||
.split('\n')
|
.split('\n')
|
||||||
.filter((x) => x.trim().length > 0)
|
.filter((x) => x.trim().length > 0)
|
||||||
.join(' ; ');
|
.map((line) => escapeForBashDoubleQuotes(line));
|
||||||
|
// Join with semicolons, but don't add semicolon after control flow keywords
|
||||||
|
// Control flow keywords that shouldn't be followed by semicolons: then, else, do, fi, done, esac
|
||||||
|
const controlFlowKeywords = /\b(then|else|do|fi|done|esac)\s*$/;
|
||||||
|
const inline = lines
|
||||||
|
.map((line, index) => {
|
||||||
|
// Don't add semicolon if this line ends with a control flow keyword
|
||||||
|
if (controlFlowKeywords.test(line.trim()) || index === lines.length - 1) {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
return `${line} ;`;
|
||||||
|
})
|
||||||
|
.join(' ');
|
||||||
const bashWrapped = `bash -lc "${inline}"`;
|
const bashWrapped = `bash -lc "${inline}"`;
|
||||||
return await cloud_runner_system_1.CloudRunnerSystem.Run(bashWrapped);
|
return await cloud_runner_system_1.CloudRunnerSystem.Run(bashWrapped);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,16 @@
|
||||||
# Provider Loader Dynamic Imports
|
# Provider Loader Dynamic Imports
|
||||||
|
|
||||||
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub repositories, and NPM packages.
|
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub
|
||||||
|
repositories, and NPM packages.
|
||||||
|
|
||||||
|
## What is a Provider?
|
||||||
|
|
||||||
|
A provider is a pluggable backend that Cloud Runner uses to run builds and workflows. Examples include AWS, Kubernetes,
|
||||||
|
or local execution. Each provider implements the `ProviderInterface`, which defines the common lifecycle methods (setup,
|
||||||
|
run, cleanup, garbage collection, etc.).
|
||||||
|
|
||||||
|
This abstraction makes Cloud Runner flexible: you can switch execution environments or add your own provider (via npm
|
||||||
|
package, GitHub repo, or local path) without changing the rest of your pipeline.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
|
|
@ -37,21 +47,18 @@ const absoluteProvider = await ProviderLoader.loadProvider('/path/to/provider',
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Load from GitHub URL
|
// Load from GitHub URL
|
||||||
const githubProvider = await ProviderLoader.loadProvider(
|
const githubProvider = await ProviderLoader.loadProvider('https://github.com/user/my-provider', buildParameters);
|
||||||
'https://github.com/user/my-provider',
|
|
||||||
buildParameters
|
|
||||||
);
|
|
||||||
|
|
||||||
// Load from specific branch
|
// Load from specific branch
|
||||||
const branchProvider = await ProviderLoader.loadProvider(
|
const branchProvider = await ProviderLoader.loadProvider(
|
||||||
'https://github.com/user/my-provider/tree/develop',
|
'https://github.com/user/my-provider/tree/develop',
|
||||||
buildParameters
|
buildParameters,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Load from specific path in repository
|
// Load from specific path in repository
|
||||||
const pathProvider = await ProviderLoader.loadProvider(
|
const pathProvider = await ProviderLoader.loadProvider(
|
||||||
'https://github.com/user/my-provider/tree/main/src/providers',
|
'https://github.com/user/my-provider/tree/main/src/providers',
|
||||||
buildParameters
|
buildParameters,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Shorthand notation
|
// Shorthand notation
|
||||||
|
|
@ -76,8 +83,20 @@ All providers must implement the `ProviderInterface`:
|
||||||
```typescript
|
```typescript
|
||||||
interface ProviderInterface {
|
interface ProviderInterface {
|
||||||
cleanupWorkflow(): Promise<void>;
|
cleanupWorkflow(): Promise<void>;
|
||||||
setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void>;
|
setupWorkflow(
|
||||||
runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string>;
|
buildGuid: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
branchName: string,
|
||||||
|
defaultSecretsArray: any[],
|
||||||
|
): Promise<void>;
|
||||||
|
runTaskInWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
task: string,
|
||||||
|
workingDirectory: string,
|
||||||
|
buildVolumeFolder: string,
|
||||||
|
environmentVariables: any[],
|
||||||
|
secrets: any[],
|
||||||
|
): Promise<string>;
|
||||||
garbageCollect(): Promise<void>;
|
garbageCollect(): Promise<void>;
|
||||||
listResources(): Promise<ProviderResource[]>;
|
listResources(): Promise<ProviderResource[]>;
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||||
|
|
@ -99,11 +118,23 @@ export default class MyProvider implements ProviderInterface {
|
||||||
// Cleanup logic
|
// Cleanup logic
|
||||||
}
|
}
|
||||||
|
|
||||||
async setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void> {
|
async setupWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
branchName: string,
|
||||||
|
defaultSecretsArray: any[],
|
||||||
|
): Promise<void> {
|
||||||
// Setup logic
|
// Setup logic
|
||||||
}
|
}
|
||||||
|
|
||||||
async runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string> {
|
async runTaskInWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
task: string,
|
||||||
|
workingDirectory: string,
|
||||||
|
buildVolumeFolder: string,
|
||||||
|
environmentVariables: any[],
|
||||||
|
secrets: any[],
|
||||||
|
): Promise<string> {
|
||||||
// Task execution logic
|
// Task execution logic
|
||||||
return 'Task completed';
|
return 'Task completed';
|
||||||
}
|
}
|
||||||
|
|
@ -159,6 +190,7 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
||||||
## Supported URL Formats
|
## Supported URL Formats
|
||||||
|
|
||||||
### GitHub URLs
|
### GitHub URLs
|
||||||
|
|
||||||
- `https://github.com/user/repo`
|
- `https://github.com/user/repo`
|
||||||
- `https://github.com/user/repo.git`
|
- `https://github.com/user/repo.git`
|
||||||
- `https://github.com/user/repo/tree/branch`
|
- `https://github.com/user/repo/tree/branch`
|
||||||
|
|
@ -166,23 +198,27 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
||||||
- `git@github.com:user/repo.git`
|
- `git@github.com:user/repo.git`
|
||||||
|
|
||||||
### Shorthand GitHub References
|
### Shorthand GitHub References
|
||||||
|
|
||||||
- `user/repo`
|
- `user/repo`
|
||||||
- `user/repo@branch`
|
- `user/repo@branch`
|
||||||
- `user/repo@branch/path/to/provider`
|
- `user/repo@branch/path/to/provider`
|
||||||
|
|
||||||
### Local Paths
|
### Local Paths
|
||||||
|
|
||||||
- `./relative/path`
|
- `./relative/path`
|
||||||
- `../relative/path`
|
- `../relative/path`
|
||||||
- `/absolute/path`
|
- `/absolute/path`
|
||||||
- `C:\\path\\to\\provider` (Windows)
|
- `C:\\path\\to\\provider` (Windows)
|
||||||
|
|
||||||
### NPM Packages
|
### NPM Packages
|
||||||
|
|
||||||
- `package-name`
|
- `package-name`
|
||||||
- `@scope/package-name`
|
- `@scope/package-name`
|
||||||
|
|
||||||
## Caching
|
## Caching
|
||||||
|
|
||||||
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the repository owner, name, and branch. This ensures that:
|
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the
|
||||||
|
repository owner, name, and branch. This ensures that:
|
||||||
|
|
||||||
1. Repositories are only cloned once
|
1. Repositories are only cloned once
|
||||||
2. Updates are checked and applied automatically
|
2. Updates are checked and applied automatically
|
||||||
|
|
@ -207,7 +243,7 @@ The provider loader can be configured through environment variables:
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
1. **Use specific branches**: Always specify the branch when loading from GitHub
|
1. **Use specific branches or versions**: Always specify the branch or specific tag when loading from GitHub
|
||||||
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
||||||
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
||||||
4. **Test locally first**: Test providers locally before deploying
|
4. **Test locally first**: Test providers locally before deploying
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,11 @@ import {
|
||||||
DescribeStacksCommand,
|
DescribeStacksCommand,
|
||||||
ListStacksCommand,
|
ListStacksCommand,
|
||||||
} from '@aws-sdk/client-cloudformation';
|
} from '@aws-sdk/client-cloudformation';
|
||||||
|
import type { ListStacksCommandOutput } from '@aws-sdk/client-cloudformation';
|
||||||
import { DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs';
|
import { DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
|
import type { DescribeLogGroupsCommandInput, DescribeLogGroupsCommandOutput } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
||||||
|
import type { DescribeTasksCommandOutput } from '@aws-sdk/client-ecs';
|
||||||
import { ListObjectsCommand } from '@aws-sdk/client-s3';
|
import { ListObjectsCommand } from '@aws-sdk/client-s3';
|
||||||
import Input from '../../../../input';
|
import Input from '../../../../input';
|
||||||
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
||||||
|
|
@ -14,6 +17,10 @@ import CloudRunner from '../../../cloud-runner';
|
||||||
import { AwsClientFactory } from '../aws-client-factory';
|
import { AwsClientFactory } from '../aws-client-factory';
|
||||||
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
||||||
|
|
||||||
|
type StackSummary = NonNullable<ListStacksCommandOutput['StackSummaries']>[number];
|
||||||
|
type LogGroup = NonNullable<DescribeLogGroupsCommandOutput['logGroups']>[number];
|
||||||
|
type Task = NonNullable<DescribeTasksCommandOutput['tasks']>[number];
|
||||||
|
|
||||||
export class TaskService {
|
export class TaskService {
|
||||||
static async watch() {
|
static async watch() {
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
|
|
@ -26,7 +33,7 @@ export class TaskService {
|
||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
public static async getCloudFormationJobStacks() {
|
public static async getCloudFormationJobStacks() {
|
||||||
const result: any[] = [];
|
const result: StackSummary[] = [];
|
||||||
CloudRunnerLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
|
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
|
|
@ -78,7 +85,12 @@ export class TaskService {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
public static async getTasks() {
|
public static async getTasks() {
|
||||||
const result: { taskElement: any; element: string }[] = [];
|
// Extended Task type to include custom properties added in this method
|
||||||
|
type ExtendedTask = Task & {
|
||||||
|
overrides?: Record<string, unknown>;
|
||||||
|
attachments?: unknown[];
|
||||||
|
};
|
||||||
|
const result: { taskElement: ExtendedTask; element: string }[] = [];
|
||||||
CloudRunnerLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
CloudRunnerLogger.log(`List Tasks`);
|
CloudRunnerLogger.log(`List Tasks`);
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
|
|
@ -102,13 +114,14 @@ export class TaskService {
|
||||||
if (taskElement === undefined) {
|
if (taskElement === undefined) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
taskElement.overrides = {};
|
const extendedTask = taskElement as ExtendedTask;
|
||||||
taskElement.attachments = [];
|
extendedTask.overrides = {};
|
||||||
if (taskElement.createdAt === undefined) {
|
extendedTask.attachments = [];
|
||||||
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
if (extendedTask.createdAt === undefined) {
|
||||||
|
CloudRunnerLogger.log(`Skipping ${extendedTask.taskDefinitionArn} no createdAt date`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
result.push({ taskElement, element });
|
result.push({ taskElement: extendedTask, element });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -149,10 +162,10 @@ export class TaskService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
public static async getLogGroups() {
|
public static async getLogGroups() {
|
||||||
const result: any[] = [];
|
const result: LogGroup[] = [];
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
const ecs = AwsClientFactory.getCloudWatchLogs();
|
const ecs = AwsClientFactory.getCloudWatchLogs();
|
||||||
let logStreamInput: any = {
|
let logStreamInput: DescribeLogGroupsCommandInput = {
|
||||||
/* logGroupNamePrefix: 'game-ci' */
|
/* logGroupNamePrefix: 'game-ci' */
|
||||||
};
|
};
|
||||||
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||||
|
|
@ -185,6 +198,7 @@ export class TaskService {
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
if (CloudRunner.buildParameters.storageProvider === 'rclone') {
|
if (CloudRunner.buildParameters.storageProvider === 'rclone') {
|
||||||
const objects = await (SharedWorkspaceLocking as any).listObjects('');
|
const objects = await (SharedWorkspaceLocking as any).listObjects('');
|
||||||
|
|
||||||
return objects.map((x: string) => ({ Key: x }));
|
return objects.map((x: string) => ({ Key: x }));
|
||||||
}
|
}
|
||||||
const s3 = AwsClientFactory.getS3();
|
const s3 = AwsClientFactory.getS3();
|
||||||
|
|
|
||||||
|
|
@ -68,13 +68,38 @@ class LocalCloudRunner implements ProviderInterface {
|
||||||
|
|
||||||
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
||||||
if (process.platform === 'win32') {
|
if (process.platform === 'win32') {
|
||||||
const inline = commands
|
// Properly escape the command string for embedding in a double-quoted bash string.
|
||||||
.replace(/"/g, '\\"')
|
// Order matters: backslashes must be escaped first to avoid double-escaping.
|
||||||
|
const escapeForBashDoubleQuotes = (stringValue: string): string => {
|
||||||
|
return stringValue
|
||||||
|
.replace(/\\/g, '\\\\') // Escape backslashes first
|
||||||
|
.replace(/\$/g, '\\$') // Escape dollar signs to prevent variable expansion
|
||||||
|
.replace(/`/g, '\\`') // Escape backticks to prevent command substitution
|
||||||
|
.replace(/"/g, '\\"'); // Escape double quotes
|
||||||
|
};
|
||||||
|
|
||||||
|
// Split commands by newlines and escape each line
|
||||||
|
const lines = commands
|
||||||
.replace(/\r/g, '')
|
.replace(/\r/g, '')
|
||||||
.split('\n')
|
.split('\n')
|
||||||
.filter((x) => x.trim().length > 0)
|
.filter((x) => x.trim().length > 0)
|
||||||
.join(' ; ');
|
.map((line) => escapeForBashDoubleQuotes(line));
|
||||||
|
|
||||||
|
// Join with semicolons, but don't add semicolon after control flow keywords
|
||||||
|
// Control flow keywords that shouldn't be followed by semicolons: then, else, do, fi, done, esac
|
||||||
|
const controlFlowKeywords = /\b(then|else|do|fi|done|esac)\s*$/;
|
||||||
|
const inline = lines
|
||||||
|
.map((line, index) => {
|
||||||
|
// Don't add semicolon if this line ends with a control flow keyword
|
||||||
|
if (controlFlowKeywords.test(line.trim()) || index === lines.length - 1) {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${line} ;`;
|
||||||
|
})
|
||||||
|
.join(' ');
|
||||||
const bashWrapped = `bash -lc "${inline}"`;
|
const bashWrapped = `bash -lc "${inline}"`;
|
||||||
|
|
||||||
return await CloudRunnerSystem.Run(bashWrapped);
|
return await CloudRunnerSystem.Run(bashWrapped);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue