actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Implement provider loader dynamic imports with GitHub URL support 

- Add URL detection and parsing utilities for GitHub URLs, local paths, and NPM packages
- Implement git operations for cloning and updating repositories with local caching
- Add automatic update checking mechanism for GitHub repositories
- Update provider-loader.ts to support multiple source types with comprehensive error handling
- Add comprehensive test coverage for all new functionality
- Include complete documentation with usage examples
- Support GitHub URLs: https://github.com/user/repo, user/repo@branch
- Support local paths: ./path, /absolute/path
- Support NPM packages: package-name, @scope/package
- Maintain backward compatibility with existing providers
- Add fallback mechanisms and interface validation
pull/734/head
Frostebite 2025-09-11 19:37:09 +01:00
parent 44bbd8c657
commit 8aa16937eb
9 changed files with 1688 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

484
dist/index.js generated vendored
View File

@ -4436,6 +4436,269 @@ class LocalCloudRunner {
exports["default"] = LocalCloudRunner;
/***/ }),
/***/ 38562:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.ProviderGitManager = void 0;
const child_process_1 = __nccwpck_require__(32081);
const util_1 = __nccwpck_require__(73837);
const fs = __importStar(__nccwpck_require__(57147));
const path_1 = __importDefault(__nccwpck_require__(71017));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const provider_url_parser_1 = __nccwpck_require__(78799);
const execAsync = (0, util_1.promisify)(child_process_1.exec);
/**
* Manages git operations for provider repositories
*/
class ProviderGitManager {
/**
* Ensures the cache directory exists
*/
static ensureCacheDir() {
if (!fs.existsSync(this.CACHE_DIR)) {
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
cloud_runner_logger_1.default.log(`Created provider cache directory: ${this.CACHE_DIR}`);
}
}
/**
* Gets the local path for a cached repository
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
static getLocalPath(urlInfo) {
const cacheKey = (0, provider_url_parser_1.generateCacheKey)(urlInfo);
return path_1.default.join(this.CACHE_DIR, cacheKey);
}
/**
* Checks if a repository is already cloned locally
* @param urlInfo GitHub URL information
* @returns True if repository exists locally
*/
static isRepositoryCloned(urlInfo) {
const localPath = this.getLocalPath(urlInfo);
return fs.existsSync(localPath) && fs.existsSync(path_1.default.join(localPath, '.git'));
}
/**
* Clones a GitHub repository to the local cache
* @param urlInfo GitHub URL information
* @returns Clone result with success status and local path
*/
static async cloneRepository(urlInfo) {
this.ensureCacheDir();
const localPath = this.getLocalPath(urlInfo);
// Remove existing directory if it exists
if (fs.existsSync(localPath)) {
cloud_runner_logger_1.default.log(`Removing existing directory: ${localPath}`);
fs.rmSync(localPath, { recursive: true, force: true });
}
try {
cloud_runner_logger_1.default.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
cloud_runner_logger_1.default.log(`Executing: ${cloneCommand}`);
const { stderr } = await execAsync(cloneCommand, {
timeout: this.GIT_TIMEOUT,
cwd: this.CACHE_DIR,
});
if (stderr && !stderr.includes('warning')) {
cloud_runner_logger_1.default.log(`Git clone stderr: ${stderr}`);
}
cloud_runner_logger_1.default.log(`Successfully cloned repository to: ${localPath}`);
return {
success: true,
localPath,
};
}
catch (error) {
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
cloud_runner_logger_1.default.log(`Error: ${errorMessage}`);
return {
success: false,
localPath,
error: errorMessage,
};
}
}
/**
* Updates a locally cloned repository
* @param urlInfo GitHub URL information
* @returns Update result with success status and whether it was updated
*/
static async updateRepository(urlInfo) {
const localPath = this.getLocalPath(urlInfo);
if (!this.isRepositoryCloned(urlInfo)) {
return {
success: false,
updated: false,
error: 'Repository not found locally',
};
}
try {
cloud_runner_logger_1.default.log(`Updating repository: ${localPath}`);
// Fetch latest changes
await execAsync('git fetch origin', {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
// Check if there are updates
const { stdout: statusOutput } = await execAsync(`git status -uno`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
const hasUpdates = statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
if (hasUpdates) {
cloud_runner_logger_1.default.log(`Updates available, pulling latest changes...`);
// Reset to origin/branch to get latest changes
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
cloud_runner_logger_1.default.log(`Repository updated successfully`);
return {
success: true,
updated: true,
};
}
else {
cloud_runner_logger_1.default.log(`Repository is already up to date`);
return {
success: true,
updated: false,
};
}
}
catch (error) {
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
cloud_runner_logger_1.default.log(`Error: ${errorMessage}`);
return {
success: false,
updated: false,
error: errorMessage,
};
}
}
/**
* Ensures a repository is available locally (clone if needed, update if exists)
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
static async ensureRepositoryAvailable(urlInfo) {
this.ensureCacheDir();
if (this.isRepositoryCloned(urlInfo)) {
cloud_runner_logger_1.default.log(`Repository already exists locally, checking for updates...`);
const updateResult = await this.updateRepository(urlInfo);
if (!updateResult.success) {
cloud_runner_logger_1.default.log(`Failed to update repository, attempting fresh clone...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
return this.getLocalPath(urlInfo);
}
else {
cloud_runner_logger_1.default.log(`Repository not found locally, cloning...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to clone repository: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
}
/**
* Gets the path to the provider module within a repository
* @param urlInfo GitHub URL information
* @param localPath Local path to the repository
* @returns Path to the provider module
*/
static getProviderModulePath(urlInfo, localPath) {
if (urlInfo.path) {
return path_1.default.join(localPath, urlInfo.path);
}
// Look for common provider entry points
const commonEntryPoints = [
'index.js',
'index.ts',
'src/index.js',
'src/index.ts',
'lib/index.js',
'lib/index.ts',
'dist/index.js',
'dist/index.js.map',
];
for (const entryPoint of commonEntryPoints) {
const fullPath = path_1.default.join(localPath, entryPoint);
if (fs.existsSync(fullPath)) {
cloud_runner_logger_1.default.log(`Found provider entry point: ${entryPoint}`);
return fullPath;
}
}
// Default to repository root
cloud_runner_logger_1.default.log(`No specific entry point found, using repository root`);
return localPath;
}
/**
* Cleans up old cached repositories (optional maintenance)
* @param maxAgeDays Maximum age in days for cached repositories
*/
static async cleanupOldRepositories(maxAgeDays = 30) {
this.ensureCacheDir();
try {
const entries = fs.readdirSync(this.CACHE_DIR, { withFileTypes: true });
const now = Date.now();
const maxAge = maxAgeDays * 24 * 60 * 60 * 1000; // Convert to milliseconds
for (const entry of entries) {
if (entry.isDirectory()) {
const entryPath = path_1.default.join(this.CACHE_DIR, entry.name);
const stats = fs.statSync(entryPath);
if (now - stats.mtime.getTime() > maxAge) {
cloud_runner_logger_1.default.log(`Cleaning up old repository: ${entry.name}`);
fs.rmSync(entryPath, { recursive: true, force: true });
}
}
}
}
catch (error) {
cloud_runner_logger_1.default.log(`Error during cleanup: ${error.message}`);
}
}
}
exports.ProviderGitManager = ProviderGitManager;
ProviderGitManager.CACHE_DIR = path_1.default.join(process.cwd(), '.provider-cache');
ProviderGitManager.GIT_TIMEOUT = 30000; // 30 seconds
/***/ }),
/***/ 45788:
@ -4472,39 +4735,80 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.ProviderLoader = void 0;
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const provider_url_parser_1 = __nccwpck_require__(78799);
const provider_git_manager_1 = __nccwpck_require__(38562);
// import path from 'path'; // Not currently used
/**
* Dynamically load a provider package by name.
* @param providerName Name of the provider package to load
* Dynamically load a provider package by name, URL, or path.
* @param providerSource Provider source (name, URL, or path)
* @param buildParameters Build parameters passed to the provider constructor
* @throws Error when the provider cannot be loaded or does not implement ProviderInterface
*/
async function loadProvider(providerName, buildParameters) {
cloud_runner_logger_1.default.log(`Loading provider: ${providerName}`);
async function loadProvider(providerSource, buildParameters) {
cloud_runner_logger_1.default.log(`Loading provider: ${providerSource}`);
// Parse the provider source to determine its type
const sourceInfo = (0, provider_url_parser_1.parseProviderSource)(providerSource);
(0, provider_url_parser_1.logProviderSource)(providerSource, sourceInfo);
let modulePath;
let importedModule;
try {
// Map provider names to their module paths for built-in providers
const providerModuleMap = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
const modulePath = providerModuleMap[providerName] || providerName;
// Handle different source types
switch (sourceInfo.type) {
case 'github': {
cloud_runner_logger_1.default.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
// Ensure the repository is available locally
const localRepoPath = await provider_git_manager_1.ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
// Get the path to the provider module within the repository
modulePath = provider_git_manager_1.ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
cloud_runner_logger_1.default.log(`Loading provider from: ${modulePath}`);
break;
}
case 'local': {
modulePath = sourceInfo.path;
cloud_runner_logger_1.default.log(`Loading provider from local path: ${modulePath}`);
break;
}
case 'npm': {
modulePath = sourceInfo.packageName;
cloud_runner_logger_1.default.log(`Loading provider from NPM package: ${modulePath}`);
break;
}
default: {
// Fallback to built-in providers or direct import
const providerModuleMap = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
modulePath = providerModuleMap[providerSource] || providerSource;
cloud_runner_logger_1.default.log(`Loading provider from module path: ${modulePath}`);
break;
}
}
// Import the module
importedModule = await Promise.resolve().then(() => __importStar(require(modulePath)));
}
catch (error) {
throw new Error(`Failed to load provider package '${providerName}': ${error.message}`);
throw new Error(`Failed to load provider package '${providerSource}': ${error.message}`);
}
// Extract the provider class/function
const Provider = importedModule.default || importedModule;
// Validate that we have a constructor
if (typeof Provider !== 'function') {
throw new TypeError(`Provider package '${providerSource}' does not export a constructor function`);
}
// Instantiate the provider
let instance;
try {
instance = new Provider(buildParameters);
}
catch (error) {
throw new Error(`Failed to instantiate provider '${providerName}': ${error.message}`);
throw new Error(`Failed to instantiate provider '${providerSource}': ${error.message}`);
}
// Validate that the instance implements the required interface
const requiredMethods = [
'cleanupWorkflow',
'setupWorkflow',
@ -4516,10 +4820,10 @@ async function loadProvider(providerName, buildParameters) {
];
for (const method of requiredMethods) {
if (typeof instance[method] !== 'function') {
throw new TypeError(`Provider package '${providerName}' does not implement ProviderInterface. Missing method '${method}'.`);
throw new TypeError(`Provider package '${providerSource}' does not implement ProviderInterface. Missing method '${method}'.`);
}
}
cloud_runner_logger_1.default.log(`Successfully loaded provider: ${providerName}`);
cloud_runner_logger_1.default.log(`Successfully loaded provider: ${providerSource}`);
return instance;
}
exports["default"] = loadProvider;
@ -4528,14 +4832,14 @@ exports["default"] = loadProvider;
*/
class ProviderLoader {
/**
* Dynamically loads a provider by name (wrapper around loadProvider function)
* @param providerName - The name of the provider to load
* Dynamically loads a provider by name, URL, or path (wrapper around loadProvider function)
* @param providerSource - The provider source (name, URL, or path) to load
* @param buildParameters - Build parameters to pass to the provider constructor
* @returns Promise<ProviderInterface> - The loaded provider instance
* @throws Error if provider package is missing or doesn't implement ProviderInterface
*/
static async loadProvider(providerName, buildParameters) {
return loadProvider(providerName, buildParameters);
static async loadProvider(providerSource, buildParameters) {
return loadProvider(providerSource, buildParameters);
}
/**
* Gets a list of available provider names
@ -4544,10 +4848,146 @@ class ProviderLoader {
static getAvailableProviders() {
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local'];
}
/**
* Cleans up old cached repositories
* @param maxAgeDays Maximum age in days for cached repositories (default: 30)
*/
static async cleanupCache(maxAgeDays = 30) {
await provider_git_manager_1.ProviderGitManager.cleanupOldRepositories(maxAgeDays);
}
/**
* Gets information about a provider source without loading it
* @param providerSource The provider source to analyze
* @returns ProviderSourceInfo object with parsed details
*/
static analyzeProviderSource(providerSource) {
return (0, provider_url_parser_1.parseProviderSource)(providerSource);
}
}
exports.ProviderLoader = ProviderLoader;
/***/ }),
/***/ 78799:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.logProviderSource = exports.isGitHubSource = exports.generateCacheKey = exports.parseProviderSource = void 0;
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
/**
* Parses a provider source string and determines its type and details
* @param source The provider source string (URL, path, or package name)
* @returns ProviderSourceInfo object with parsed details
*/
function parseProviderSource(source) {
// Check if it's a GitHub URL
const githubMatch = source.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
if (githubMatch) {
const [, owner, repo, branch, path] = githubMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: source,
};
}
// Check if it's a GitHub SSH URL
const githubSshMatch = source.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
if (githubSshMatch) {
const [, owner, repo, branch, path] = githubSshMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a shorthand GitHub reference (owner/repo)
const shorthandMatch = source.match(/^([^/@]+)\/([^/@]+)(?:@([^/]+))?(?:\/(.+))?$/);
if (shorthandMatch && !source.startsWith('.') && !source.startsWith('/') && !source.includes('\\')) {
const [, owner, repo, branch, path] = shorthandMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a local path
if (source.startsWith('./') || source.startsWith('../') || source.startsWith('/') || source.includes('\\')) {
return {
type: 'local',
path: source,
};
}
// Default to npm package
return {
type: 'npm',
packageName: source,
};
}
exports.parseProviderSource = parseProviderSource;
/**
* Generates a cache key for a GitHub repository
* @param urlInfo GitHub URL information
* @returns Cache key string
*/
function generateCacheKey(urlInfo) {
return `github_${urlInfo.owner}_${urlInfo.repo}_${urlInfo.branch}`.replace(/[^\w-]/g, '_');
}
exports.generateCacheKey = generateCacheKey;
/**
* Validates if a string looks like a valid GitHub URL or reference
* @param source The source string to validate
* @returns True if it looks like a GitHub reference
*/
function isGitHubSource(source) {
const parsed = parseProviderSource(source);
return parsed.type === 'github';
}
exports.isGitHubSource = isGitHubSource;
/**
* Logs the parsed provider source information
* @param source The original source string
* @param parsed The parsed source information
*/
function logProviderSource(source, parsed) {
cloud_runner_logger_1.default.log(`Provider source: ${source}`);
switch (parsed.type) {
case 'github':
cloud_runner_logger_1.default.log(` Type: GitHub repository`);
cloud_runner_logger_1.default.log(` Owner: ${parsed.owner}`);
cloud_runner_logger_1.default.log(` Repository: ${parsed.repo}`);
cloud_runner_logger_1.default.log(` Branch: ${parsed.branch}`);
if (parsed.path) {
cloud_runner_logger_1.default.log(` Path: ${parsed.path}`);
}
break;
case 'local':
cloud_runner_logger_1.default.log(` Type: Local path`);
cloud_runner_logger_1.default.log(` Path: ${parsed.path}`);
break;
case 'npm':
cloud_runner_logger_1.default.log(` Type: NPM package`);
cloud_runner_logger_1.default.log(` Package: ${parsed.packageName}`);
break;
}
}
exports.logProviderSource = logProviderSource;
/***/ }),
/***/ 63007:

2
dist/index.js.map generated vendored
View File

File diff suppressed because one or more lines are too long

214
src/model/cloud-runner/providers/README.md 100644
View File

@ -0,0 +1,214 @@
# Provider Loader Dynamic Imports
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub repositories, and NPM packages.
## Features
- **Local File Paths**: Load providers from relative or absolute file paths
- **GitHub URLs**: Clone and load providers from GitHub repositories with automatic updates
- **NPM Packages**: Load providers from installed NPM packages
- **Automatic Updates**: GitHub repositories are automatically updated when changes are available
- **Caching**: Local caching of cloned repositories for improved performance
- **Fallback Support**: Graceful fallback to local provider if loading fails
## Usage Examples
### Loading Built-in Providers
```typescript
import { ProviderLoader } from './provider-loader';
// Load built-in providers
const awsProvider = await ProviderLoader.loadProvider('aws', buildParameters);
const k8sProvider = await ProviderLoader.loadProvider('k8s', buildParameters);
```
### Loading Local Providers
```typescript
// Load from relative path
const localProvider = await ProviderLoader.loadProvider('./my-local-provider', buildParameters);
// Load from absolute path
const absoluteProvider = await ProviderLoader.loadProvider('/path/to/provider', buildParameters);
```
### Loading GitHub Providers
```typescript
// Load from GitHub URL
const githubProvider = await ProviderLoader.loadProvider(
'https://github.com/user/my-provider',
buildParameters
);
// Load from specific branch
const branchProvider = await ProviderLoader.loadProvider(
'https://github.com/user/my-provider/tree/develop',
buildParameters
);
// Load from specific path in repository
const pathProvider = await ProviderLoader.loadProvider(
'https://github.com/user/my-provider/tree/main/src/providers',
buildParameters
);
// Shorthand notation
const shorthandProvider = await ProviderLoader.loadProvider('user/repo', buildParameters);
const branchShorthand = await ProviderLoader.loadProvider('user/repo@develop', buildParameters);
```
### Loading NPM Packages
```typescript
// Load from NPM package
const npmProvider = await ProviderLoader.loadProvider('my-provider-package', buildParameters);
// Load from scoped NPM package
const scopedProvider = await ProviderLoader.loadProvider('@scope/my-provider', buildParameters);
```
## Provider Interface
All providers must implement the `ProviderInterface`:
```typescript
interface ProviderInterface {
cleanupWorkflow(): Promise<void>;
setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void>;
runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string>;
garbageCollect(): Promise<void>;
listResources(): Promise<ProviderResource[]>;
listWorkflow(): Promise<ProviderWorkflow[]>;
watchWorkflow(): Promise<void>;
}
```
## Example Provider Implementation
```typescript
// my-provider.ts
import { ProviderInterface } from './provider-interface';
import BuildParameters from './build-parameters';
export default class MyProvider implements ProviderInterface {
constructor(private buildParameters: BuildParameters) {}
async cleanupWorkflow(): Promise<void> {
// Cleanup logic
}
async setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void> {
// Setup logic
}
async runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string> {
// Task execution logic
return 'Task completed';
}
async garbageCollect(): Promise<void> {
// Garbage collection logic
}
async listResources(): Promise<ProviderResource[]> {
return [];
}
async listWorkflow(): Promise<ProviderWorkflow[]> {
return [];
}
async watchWorkflow(): Promise<void> {
// Watch logic
}
}
```
## Utility Methods
### Analyze Provider Source
```typescript
// Analyze a provider source without loading it
const sourceInfo = ProviderLoader.analyzeProviderSource('https://github.com/user/repo');
console.log(sourceInfo.type); // 'github'
console.log(sourceInfo.owner); // 'user'
console.log(sourceInfo.repo); // 'repo'
```
### Clean Up Cache
```typescript
// Clean up old cached repositories (older than 30 days)
await ProviderLoader.cleanupCache();
// Clean up repositories older than 7 days
await ProviderLoader.cleanupCache(7);
```
### Get Available Providers
```typescript
// Get list of built-in providers
const providers = ProviderLoader.getAvailableProviders();
console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local']
```
## Supported URL Formats
### GitHub URLs
- `https://github.com/user/repo`
- `https://github.com/user/repo.git`
- `https://github.com/user/repo/tree/branch`
- `https://github.com/user/repo/tree/branch/path/to/provider`
- `git@github.com:user/repo.git`
### Shorthand GitHub References
- `user/repo`
- `user/repo@branch`
- `user/repo@branch/path/to/provider`
### Local Paths
- `./relative/path`
- `../relative/path`
- `/absolute/path`
- `C:\\path\\to\\provider` (Windows)
### NPM Packages
- `package-name`
- `@scope/package-name`
## Caching
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the repository owner, name, and branch. This ensures that:
1. Repositories are only cloned once
2. Updates are checked and applied automatically
3. Performance is improved for repeated loads
4. Storage is managed efficiently
## Error Handling
The provider loader includes comprehensive error handling:
- **Missing packages**: Clear error messages when providers cannot be found
- **Interface validation**: Ensures providers implement the required interface
- **Git operations**: Handles network issues and repository access problems
- **Fallback mechanism**: Falls back to local provider if loading fails
## Configuration
The provider loader can be configured through environment variables:
- `PROVIDER_CACHE_DIR`: Custom cache directory (default: `.provider-cache`)
- `GIT_TIMEOUT`: Git operation timeout in milliseconds (default: 30000)
## Best Practices
1. **Use specific branches**: Always specify the branch when loading from GitHub
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
3. **Clean up regularly**: Use the cleanup utility to manage cache size
4. **Test locally first**: Test providers locally before deploying
5. **Use semantic versioning**: Tag your provider repositories for stable versions

213
src/model/cloud-runner/providers/provider-git-manager.test.ts 100644
View File

@ -0,0 +1,213 @@
import { ProviderGitManager } from './provider-git-manager';
import { GitHubUrlInfo } from './provider-url-parser';
import * as fs from 'fs';
import path from 'path';
import { exec } from 'child_process';
// Mock the exec function
jest.mock('child_process', () => ({
exec: jest.fn(),
}));
// Mock fs module
jest.mock('fs');
const mockFs = fs as jest.Mocked<typeof fs>;
const mockExec = exec as jest.MockedFunction<typeof exec>;
describe('ProviderGitManager', () => {
const mockUrlInfo: GitHubUrlInfo = {
type: 'github',
owner: 'test-user',
repo: 'test-repo',
branch: 'main',
url: 'https://github.com/test-user/test-repo',
};
beforeEach(() => {
jest.clearAllMocks();
mockExec.mockImplementation((command, options, callback) => {
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
});
});
describe('isRepositoryCloned', () => {
it('returns true when repository exists', () => {
const localPath = ProviderGitManager['getLocalPath'](mockUrlInfo);
mockFs.existsSync.mockReturnValue(true);
const result = ProviderGitManager['isRepositoryCloned'](mockUrlInfo);
expect(result).toBe(true);
expect(mockFs.existsSync).toHaveBeenCalledWith(localPath);
expect(mockFs.existsSync).toHaveBeenCalledWith(path.join(localPath, '.git'));
});
it('returns false when repository does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
const result = ProviderGitManager['isRepositoryCloned'](mockUrlInfo);
expect(result).toBe(false);
});
});
describe('cloneRepository', () => {
it('successfully clones a repository', async () => {
mockFs.existsSync.mockReturnValue(false);
mockFs.mkdirSync.mockImplementation(() => '');
mockFs.rmSync.mockImplementation(() => {});
const result = await ProviderGitManager.cloneRepository(mockUrlInfo);
expect(result.success).toBe(true);
expect(result.localPath).toContain('github_test-user_test-repo_main');
expect(mockExec).toHaveBeenCalledWith(
expect.stringContaining('git clone'),
expect.objectContaining({
timeout: 30000,
cwd: expect.any(String),
}),
);
});
it('handles clone errors', async () => {
mockFs.existsSync.mockReturnValue(false);
mockFs.mkdirSync.mockImplementation(() => '');
mockFs.rmSync.mockImplementation(() => {});
mockExec.mockImplementation((command, options, callback) => {
if (callback) {
callback(new Error('Clone failed'), '', 'error');
}
return { stdout: '', stderr: 'error' } as any;
});
const result = await ProviderGitManager.cloneRepository(mockUrlInfo);
expect(result.success).toBe(false);
expect(result.error).toContain('Clone failed');
});
});
describe('updateRepository', () => {
it('successfully updates a repository when updates are available', async () => {
mockFs.existsSync.mockReturnValue(true);
// Mock git status output indicating updates are available
mockExec.mockImplementation((command, options, callback) => {
if (command.includes('git status')) {
if (callback) {
callback(undefined as any, 'Your branch is behind', '');
}
return { stdout: 'Your branch is behind', stderr: '' } as any;
} else if (command.includes('git reset')) {
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
} else if (command.includes('git fetch')) {
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
}
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
});
const result = await ProviderGitManager.updateRepository(mockUrlInfo);
expect(result.success).toBe(true);
expect(result.updated).toBe(true);
});
it('reports no updates when repository is up to date', async () => {
mockFs.existsSync.mockReturnValue(true);
// Mock git status output indicating no updates
mockExec.mockImplementation((command, options, callback) => {
if (command.includes('git status')) {
if (callback) {
callback(undefined as any, 'Your branch is up to date', '');
}
return { stdout: 'Your branch is up to date', stderr: '' } as any;
} else if (command.includes('git fetch')) {
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
}
if (callback) {
callback(undefined as any, 'success', '');
}
return { stdout: 'success', stderr: '' } as any;
});
const result = await ProviderGitManager.updateRepository(mockUrlInfo);
expect(result.success).toBe(true);
expect(result.updated).toBe(false);
});
it('handles update errors', async () => {
mockFs.existsSync.mockReturnValue(true);
mockExec.mockImplementation((command, options, callback) => {
if (callback) {
callback(new Error('Update failed'), '', 'error');
}
return { stdout: '', stderr: 'error' } as any;
});
const result = await ProviderGitManager.updateRepository(mockUrlInfo);
expect(result.success).toBe(false);
expect(result.error).toContain('Update failed');
});
});
describe('getProviderModulePath', () => {
it('returns the specified path when provided', () => {
const urlInfoWithPath = { ...mockUrlInfo, path: 'src/providers' };
const localPath = '/path/to/repo';
const result = ProviderGitManager.getProviderModulePath(urlInfoWithPath, localPath);
expect(result).toBe(path.join(localPath, 'src/providers'));
});
it('finds common entry points when no path specified', () => {
const localPath = '/path/to/repo';
mockFs.existsSync.mockImplementation((filePath) => {
return filePath === path.join(localPath, 'index.js');
});
const result = ProviderGitManager.getProviderModulePath(mockUrlInfo, localPath);
expect(result).toBe(path.join(localPath, 'index.js'));
});
it('returns repository root when no entry point found', () => {
const localPath = '/path/to/repo';
mockFs.existsSync.mockReturnValue(false);
const result = ProviderGitManager.getProviderModulePath(mockUrlInfo, localPath);
expect(result).toBe(localPath);
});
});
});

278
src/model/cloud-runner/providers/provider-git-manager.ts 100644
View File

@ -0,0 +1,278 @@
import { exec } from 'child_process';
import { promisify } from 'util';
import * as fs from 'fs';
import path from 'path';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser';
const execAsync = promisify(exec);
export interface GitCloneResult {
success: boolean;
localPath: string;
error?: string;
}
export interface GitUpdateResult {
success: boolean;
updated: boolean;
error?: string;
}
/**
* Manages git operations for provider repositories
*/
export class ProviderGitManager {
private static readonly CACHE_DIR = path.join(process.cwd(), '.provider-cache');
private static readonly GIT_TIMEOUT = 30000; // 30 seconds
/**
* Ensures the cache directory exists
*/
private static ensureCacheDir(): void {
if (!fs.existsSync(this.CACHE_DIR)) {
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
CloudRunnerLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`);
}
}
/**
* Gets the local path for a cached repository
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
private static getLocalPath(urlInfo: GitHubUrlInfo): string {
const cacheKey = generateCacheKey(urlInfo);
return path.join(this.CACHE_DIR, cacheKey);
}
/**
* Checks if a repository is already cloned locally
* @param urlInfo GitHub URL information
* @returns True if repository exists locally
*/
private static isRepositoryCloned(urlInfo: GitHubUrlInfo): boolean {
const localPath = this.getLocalPath(urlInfo);
return fs.existsSync(localPath) && fs.existsSync(path.join(localPath, '.git'));
}
/**
* Clones a GitHub repository to the local cache
* @param urlInfo GitHub URL information
* @returns Clone result with success status and local path
*/
static async cloneRepository(urlInfo: GitHubUrlInfo): Promise<GitCloneResult> {
this.ensureCacheDir();
const localPath = this.getLocalPath(urlInfo);
// Remove existing directory if it exists
if (fs.existsSync(localPath)) {
CloudRunnerLogger.log(`Removing existing directory: ${localPath}`);
fs.rmSync(localPath, { recursive: true, force: true });
}
try {
CloudRunnerLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
CloudRunnerLogger.log(`Executing: ${cloneCommand}`);
const { stderr } = await execAsync(cloneCommand, {
timeout: this.GIT_TIMEOUT,
cwd: this.CACHE_DIR,
});
if (stderr && !stderr.includes('warning')) {
CloudRunnerLogger.log(`Git clone stderr: ${stderr}`);
}
CloudRunnerLogger.log(`Successfully cloned repository to: ${localPath}`);
return {
success: true,
localPath,
};
} catch (error: any) {
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
CloudRunnerLogger.log(`Error: ${errorMessage}`);
return {
success: false,
localPath,
error: errorMessage,
};
}
}
/**
* Updates a locally cloned repository
* @param urlInfo GitHub URL information
* @returns Update result with success status and whether it was updated
*/
static async updateRepository(urlInfo: GitHubUrlInfo): Promise<GitUpdateResult> {
const localPath = this.getLocalPath(urlInfo);
if (!this.isRepositoryCloned(urlInfo)) {
return {
success: false,
updated: false,
error: 'Repository not found locally',
};
}
try {
CloudRunnerLogger.log(`Updating repository: ${localPath}`);
// Fetch latest changes
await execAsync('git fetch origin', {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
// Check if there are updates
const { stdout: statusOutput } = await execAsync(`git status -uno`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
const hasUpdates =
statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
if (hasUpdates) {
CloudRunnerLogger.log(`Updates available, pulling latest changes...`);
// Reset to origin/branch to get latest changes
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
CloudRunnerLogger.log(`Repository updated successfully`);
return {
success: true,
updated: true,
};
} else {
CloudRunnerLogger.log(`Repository is already up to date`);
return {
success: true,
updated: false,
};
}
} catch (error: any) {
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
CloudRunnerLogger.log(`Error: ${errorMessage}`);
return {
success: false,
updated: false,
error: errorMessage,
};
}
}
/**
* Ensures a repository is available locally (clone if needed, update if exists)
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
static async ensureRepositoryAvailable(urlInfo: GitHubUrlInfo): Promise<string> {
this.ensureCacheDir();
if (this.isRepositoryCloned(urlInfo)) {
CloudRunnerLogger.log(`Repository already exists locally, checking for updates...`);
const updateResult = await this.updateRepository(urlInfo);
if (!updateResult.success) {
CloudRunnerLogger.log(`Failed to update repository, attempting fresh clone...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
return this.getLocalPath(urlInfo);
} else {
CloudRunnerLogger.log(`Repository not found locally, cloning...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to clone repository: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
}
/**
* Gets the path to the provider module within a repository
* @param urlInfo GitHub URL information
* @param localPath Local path to the repository
* @returns Path to the provider module
*/
static getProviderModulePath(urlInfo: GitHubUrlInfo, localPath: string): string {
if (urlInfo.path) {
return path.join(localPath, urlInfo.path);
}
// Look for common provider entry points
const commonEntryPoints = [
'index.js',
'index.ts',
'src/index.js',
'src/index.ts',
'lib/index.js',
'lib/index.ts',
'dist/index.js',
'dist/index.js.map',
];
for (const entryPoint of commonEntryPoints) {
const fullPath = path.join(localPath, entryPoint);
if (fs.existsSync(fullPath)) {
CloudRunnerLogger.log(`Found provider entry point: ${entryPoint}`);
return fullPath;
}
}
// Default to repository root
CloudRunnerLogger.log(`No specific entry point found, using repository root`);
return localPath;
}
/**
* Cleans up old cached repositories (optional maintenance)
* @param maxAgeDays Maximum age in days for cached repositories
*/
static async cleanupOldRepositories(maxAgeDays: number = 30): Promise<void> {
this.ensureCacheDir();
try {
const entries = fs.readdirSync(this.CACHE_DIR, { withFileTypes: true });
const now = Date.now();
const maxAge = maxAgeDays * 24 * 60 * 60 * 1000; // Convert to milliseconds
for (const entry of entries) {
if (entry.isDirectory()) {
const entryPath = path.join(this.CACHE_DIR, entry.name);
const stats = fs.statSync(entryPath);
if (now - stats.mtime.getTime() > maxAge) {
CloudRunnerLogger.log(`Cleaning up old repository: ${entry.name}`);
fs.rmSync(entryPath, { recursive: true, force: true });
}
}
}
} catch (error: any) {
CloudRunnerLogger.log(`Error during cleanup: ${error.message}`);
}
}
}

114
src/model/cloud-runner/providers/provider-loader.test.ts
View File

@ -1,19 +1,113 @@
import loadProvider from './provider-loader';
import loadProvider, { ProviderLoader } from './provider-loader';
import { ProviderInterface } from './provider-interface';
import { ProviderGitManager } from './provider-git-manager';
// Mock the git manager
jest.mock('./provider-git-manager');
const mockProviderGitManager = ProviderGitManager as jest.Mocked<typeof ProviderGitManager>;
describe('provider-loader', () => {
it('loads a provider dynamically', async () => {
const provider: ProviderInterface = await loadProvider('./test', {} as any);
expect(typeof provider.runTaskInWorkflow).toBe('function');
beforeEach(() => {
jest.clearAllMocks();
});
it('throws when provider package is missing', async () => {
await expect(loadProvider('non-existent-package', {} as any)).rejects.toThrow('non-existent-package');
describe('loadProvider', () => {
it('loads a built-in provider dynamically', async () => {
const provider: ProviderInterface = await loadProvider('test', {} as any);
expect(typeof provider.runTaskInWorkflow).toBe('function');
});
it('loads a local provider from relative path', async () => {
const provider: ProviderInterface = await loadProvider('./test', {} as any);
expect(typeof provider.runTaskInWorkflow).toBe('function');
});
it('loads a GitHub provider', async () => {
const mockLocalPath = '/path/to/cloned/repo';
const mockModulePath = '/path/to/cloned/repo/index.js';
mockProviderGitManager.ensureRepositoryAvailable.mockResolvedValue(mockLocalPath);
mockProviderGitManager.getProviderModulePath.mockReturnValue(mockModulePath);
// Mock the import to return a valid provider
const mockProvider = {
default: class MockProvider {
constructor() {}
runTaskInWorkflow() {}
cleanupWorkflow() {}
setupWorkflow() {}
garbageCollect() {}
listResources() {}
listWorkflow() {}
watchWorkflow() {}
},
};
jest.doMock(mockModulePath, () => mockProvider, { virtual: true });
const provider: ProviderInterface = await loadProvider('https://github.com/user/repo', {} as any);
expect(typeof provider.runTaskInWorkflow).toBe('function');
expect(mockProviderGitManager.ensureRepositoryAvailable).toHaveBeenCalled();
});
it('throws when provider package is missing', async () => {
await expect(loadProvider('non-existent-package', {} as any)).rejects.toThrow('non-existent-package');
});
it('throws when provider does not implement ProviderInterface', async () => {
await expect(loadProvider('./fixtures/invalid-provider', {} as any)).rejects.toThrow(
'does not implement ProviderInterface',
);
});
it('throws when provider does not export a constructor', async () => {
// Mock a module that doesn't export a constructor
jest.doMock('./test', () => ({ default: 'not-a-constructor' }), { virtual: true });
await expect(loadProvider('./test', {} as any)).rejects.toThrow('does not export a constructor function');
});
});
it('throws when provider does not implement ProviderInterface', async () => {
await expect(loadProvider('./fixtures/invalid-provider', {} as any)).rejects.toThrow(
'does not implement ProviderInterface',
);
describe('ProviderLoader class', () => {
it('loads providers using the static method', async () => {
const provider: ProviderInterface = await ProviderLoader.loadProvider('test', {} as any);
expect(typeof provider.runTaskInWorkflow).toBe('function');
});
it('returns available providers', () => {
const providers = ProviderLoader.getAvailableProviders();
expect(providers).toContain('aws');
expect(providers).toContain('k8s');
expect(providers).toContain('test');
});
it('cleans up cache', async () => {
mockProviderGitManager.cleanupOldRepositories.mockResolvedValue();
await ProviderLoader.cleanupCache(7);
expect(mockProviderGitManager.cleanupOldRepositories).toHaveBeenCalledWith(7);
});
it('analyzes provider sources', () => {
const githubInfo = ProviderLoader.analyzeProviderSource('https://github.com/user/repo');
expect(githubInfo.type).toBe('github');
if (githubInfo.type === 'github') {
expect(githubInfo.owner).toBe('user');
expect(githubInfo.repo).toBe('repo');
}
const localInfo = ProviderLoader.analyzeProviderSource('./local-provider');
expect(localInfo.type).toBe('local');
if (localInfo.type === 'local') {
expect(localInfo.path).toBe('./local-provider');
}
const npmInfo = ProviderLoader.analyzeProviderSource('my-package');
expect(npmInfo.type).toBe('npm');
if (npmInfo.type === 'npm') {
expect(npmInfo.packageName).toBe('my-package');
}
});
});
});

116
src/model/cloud-runner/providers/provider-loader.ts
View File

@ -1,45 +1,98 @@
import { ProviderInterface } from './provider-interface';
import BuildParameters from '../../build-parameters';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser';
import { ProviderGitManager } from './provider-git-manager';
// import path from 'path'; // Not currently used
/**
* Dynamically load a provider package by name.
* @param providerName Name of the provider package to load
* Dynamically load a provider package by name, URL, or path.
* @param providerSource Provider source (name, URL, or path)
* @param buildParameters Build parameters passed to the provider constructor
* @throws Error when the provider cannot be loaded or does not implement ProviderInterface
*/
export default async function loadProvider(
providerName: string,
providerSource: string,
buildParameters: BuildParameters,
): Promise<ProviderInterface> {
CloudRunnerLogger.log(`Loading provider: ${providerName}`);
CloudRunnerLogger.log(`Loading provider: ${providerSource}`);
// Parse the provider source to determine its type
const sourceInfo = parseProviderSource(providerSource);
logProviderSource(providerSource, sourceInfo);
let modulePath: string;
let importedModule: any;
try {
// Map provider names to their module paths for built-in providers
const providerModuleMap: Record<string, string> = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
const modulePath = providerModuleMap[providerName] || providerName;
try {
// Handle different source types
switch (sourceInfo.type) {
case 'github': {
CloudRunnerLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
// Ensure the repository is available locally
const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
// Get the path to the provider module within the repository
modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
CloudRunnerLogger.log(`Loading provider from: ${modulePath}`);
break;
}
case 'local': {
modulePath = sourceInfo.path;
CloudRunnerLogger.log(`Loading provider from local path: ${modulePath}`);
break;
}
case 'npm': {
modulePath = sourceInfo.packageName;
CloudRunnerLogger.log(`Loading provider from NPM package: ${modulePath}`);
break;
}
default: {
// Fallback to built-in providers or direct import
const providerModuleMap: Record<string, string> = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
modulePath = providerModuleMap[providerSource] || providerSource;
CloudRunnerLogger.log(`Loading provider from module path: ${modulePath}`);
break;
}
}
// Import the module
importedModule = await import(modulePath);
} catch (error) {
throw new Error(`Failed to load provider package '${providerName}': ${(error as Error).message}`);
throw new Error(`Failed to load provider package '${providerSource}': ${(error as Error).message}`);
}
// Extract the provider class/function
const Provider = importedModule.default || importedModule;
// Validate that we have a constructor
if (typeof Provider !== 'function') {
throw new TypeError(`Provider package '${providerSource}' does not export a constructor function`);
}
// Instantiate the provider
let instance: any;
try {
instance = new Provider(buildParameters);
} catch (error) {
throw new Error(`Failed to instantiate provider '${providerName}': ${(error as Error).message}`);
throw new Error(`Failed to instantiate provider '${providerSource}': ${(error as Error).message}`);
}
// Validate that the instance implements the required interface
const requiredMethods = [
'cleanupWorkflow',
'setupWorkflow',
@ -53,12 +106,12 @@ export default async function loadProvider(
for (const method of requiredMethods) {
if (typeof instance[method] !== 'function') {
throw new TypeError(
`Provider package '${providerName}' does not implement ProviderInterface. Missing method '${method}'.`,
`Provider package '${providerSource}' does not implement ProviderInterface. Missing method '${method}'.`,
);
}
}
CloudRunnerLogger.log(`Successfully loaded provider: ${providerName}`);
CloudRunnerLogger.log(`Successfully loaded provider: ${providerSource}`);
return instance as ProviderInterface;
}
@ -68,14 +121,14 @@ export default async function loadProvider(
*/
export class ProviderLoader {
/**
* Dynamically loads a provider by name (wrapper around loadProvider function)
* @param providerName - The name of the provider to load
* Dynamically loads a provider by name, URL, or path (wrapper around loadProvider function)
* @param providerSource - The provider source (name, URL, or path) to load
* @param buildParameters - Build parameters to pass to the provider constructor
* @returns Promise<ProviderInterface> - The loaded provider instance
* @throws Error if provider package is missing or doesn't implement ProviderInterface
*/
static async loadProvider(providerName: string, buildParameters: BuildParameters): Promise<ProviderInterface> {
return loadProvider(providerName, buildParameters);
static async loadProvider(providerSource: string, buildParameters: BuildParameters): Promise<ProviderInterface> {
return loadProvider(providerSource, buildParameters);
}
/**
@ -85,4 +138,21 @@ export class ProviderLoader {
static getAvailableProviders(): string[] {
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local'];
}
/**
* Cleans up old cached repositories
* @param maxAgeDays Maximum age in days for cached repositories (default: 30)
*/
static async cleanupCache(maxAgeDays: number = 30): Promise<void> {
await ProviderGitManager.cleanupOldRepositories(maxAgeDays);
}
/**
* Gets information about a provider source without loading it
* @param providerSource The provider source to analyze
* @returns ProviderSourceInfo object with parsed details
*/
static analyzeProviderSource(providerSource: string): ProviderSourceInfo {
return parseProviderSource(providerSource);
}
}

185
src/model/cloud-runner/providers/provider-url-parser.test.ts 100644
View File

@ -0,0 +1,185 @@
import { parseProviderSource, generateCacheKey, isGitHubSource } from './provider-url-parser';
describe('provider-url-parser', () => {
describe('parseProviderSource', () => {
it('parses HTTPS GitHub URLs correctly', () => {
const result = parseProviderSource('https://github.com/user/repo');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses HTTPS GitHub URLs with branch', () => {
const result = parseProviderSource('https://github.com/user/repo/tree/develop');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'develop',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses HTTPS GitHub URLs with path', () => {
const result = parseProviderSource('https://github.com/user/repo/tree/main/src/providers');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: 'src/providers',
url: 'https://github.com/user/repo',
});
});
it('parses GitHub URLs with .git extension', () => {
const result = parseProviderSource('https://github.com/user/repo.git');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses SSH GitHub URLs', () => {
const result = parseProviderSource('git@github.com:user/repo.git');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses shorthand GitHub references', () => {
const result = parseProviderSource('user/repo');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses shorthand GitHub references with branch', () => {
const result = parseProviderSource('user/repo@develop');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'develop',
path: '',
url: 'https://github.com/user/repo',
});
});
it('parses shorthand GitHub references with path', () => {
const result = parseProviderSource('user/repo@main/src/providers');
expect(result).toEqual({
type: 'github',
owner: 'user',
repo: 'repo',
branch: 'main',
path: 'src/providers',
url: 'https://github.com/user/repo',
});
});
it('parses local relative paths', () => {
const result = parseProviderSource('./my-provider');
expect(result).toEqual({
type: 'local',
path: './my-provider',
});
});
it('parses local absolute paths', () => {
const result = parseProviderSource('/path/to/provider');
expect(result).toEqual({
type: 'local',
path: '/path/to/provider',
});
});
it('parses Windows paths', () => {
const result = parseProviderSource('C:\\path\\to\\provider');
expect(result).toEqual({
type: 'local',
path: 'C:\\path\\to\\provider',
});
});
it('parses NPM package names', () => {
const result = parseProviderSource('my-provider-package');
expect(result).toEqual({
type: 'npm',
packageName: 'my-provider-package',
});
});
it('parses scoped NPM package names', () => {
const result = parseProviderSource('@scope/my-provider');
expect(result).toEqual({
type: 'npm',
packageName: '@scope/my-provider',
});
});
});
describe('generateCacheKey', () => {
it('generates valid cache keys for GitHub URLs', () => {
const urlInfo = {
type: 'github' as const,
owner: 'user',
repo: 'my-repo',
branch: 'develop',
url: 'https://github.com/user/my-repo',
};
const key = generateCacheKey(urlInfo);
expect(key).toBe('github_user_my-repo_develop');
});
it('handles special characters in cache keys', () => {
const urlInfo = {
type: 'github' as const,
owner: 'user-name',
repo: 'my.repo',
branch: 'feature/branch',
url: 'https://github.com/user-name/my.repo',
};
const key = generateCacheKey(urlInfo);
expect(key).toBe('github_user-name_my_repo_feature_branch');
});
});
describe('isGitHubSource', () => {
it('identifies GitHub URLs correctly', () => {
expect(isGitHubSource('https://github.com/user/repo')).toBe(true);
expect(isGitHubSource('git@github.com:user/repo.git')).toBe(true);
expect(isGitHubSource('user/repo')).toBe(true);
expect(isGitHubSource('user/repo@develop')).toBe(true);
});
it('identifies non-GitHub sources correctly', () => {
expect(isGitHubSource('./local-provider')).toBe(false);
expect(isGitHubSource('/absolute/path')).toBe(false);
expect(isGitHubSource('npm-package')).toBe(false);
expect(isGitHubSource('@scope/package')).toBe(false);
});
});
});

138
src/model/cloud-runner/providers/provider-url-parser.ts 100644
View File

@ -0,0 +1,138 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
export interface GitHubUrlInfo {
type: 'github';
owner: string;
repo: string;
branch?: string;
path?: string;
url: string;
}
export interface LocalPathInfo {
type: 'local';
path: string;
}
export interface NpmPackageInfo {
type: 'npm';
packageName: string;
}
export type ProviderSourceInfo = GitHubUrlInfo | LocalPathInfo | NpmPackageInfo;
/**
* Parses a provider source string and determines its type and details
* @param source The provider source string (URL, path, or package name)
* @returns ProviderSourceInfo object with parsed details
*/
export function parseProviderSource(source: string): ProviderSourceInfo {
// Check if it's a GitHub URL
const githubMatch = source.match(
/^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/,
);
if (githubMatch) {
const [, owner, repo, branch, path] = githubMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: source,
};
}
// Check if it's a GitHub SSH URL
const githubSshMatch = source.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
if (githubSshMatch) {
const [, owner, repo, branch, path] = githubSshMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a shorthand GitHub reference (owner/repo)
const shorthandMatch = source.match(/^([^/@]+)\/([^/@]+)(?:@([^/]+))?(?:\/(.+))?$/);
if (shorthandMatch && !source.startsWith('.') && !source.startsWith('/') && !source.includes('\\')) {
const [, owner, repo, branch, path] = shorthandMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a local path
if (source.startsWith('./') || source.startsWith('../') || source.startsWith('/') || source.includes('\\')) {
return {
type: 'local',
path: source,
};
}
// Default to npm package
return {
type: 'npm',
packageName: source,
};
}
/**
* Generates a cache key for a GitHub repository
* @param urlInfo GitHub URL information
* @returns Cache key string
*/
export function generateCacheKey(urlInfo: GitHubUrlInfo): string {
return `github_${urlInfo.owner}_${urlInfo.repo}_${urlInfo.branch}`.replace(/[^\w-]/g, '_');
}
/**
* Validates if a string looks like a valid GitHub URL or reference
* @param source The source string to validate
* @returns True if it looks like a GitHub reference
*/
export function isGitHubSource(source: string): boolean {
const parsed = parseProviderSource(source);
return parsed.type === 'github';
}
/**
* Logs the parsed provider source information
* @param source The original source string
* @param parsed The parsed source information
*/
export function logProviderSource(source: string, parsed: ProviderSourceInfo): void {
CloudRunnerLogger.log(`Provider source: ${source}`);
switch (parsed.type) {
case 'github':
CloudRunnerLogger.log(` Type: GitHub repository`);
CloudRunnerLogger.log(` Owner: ${parsed.owner}`);
CloudRunnerLogger.log(` Repository: ${parsed.repo}`);
CloudRunnerLogger.log(` Branch: ${parsed.branch}`);
if (parsed.path) {
CloudRunnerLogger.log(` Path: ${parsed.path}`);
}
break;
case 'local':
CloudRunnerLogger.log(` Type: Local path`);
CloudRunnerLogger.log(` Path: ${parsed.path}`);
break;
case 'npm':
CloudRunnerLogger.log(` Type: NPM package`);
CloudRunnerLogger.log(` Package: ${parsed.packageName}`);
break;
}
}