feat: Implement provider loader dynamic imports with GitHub URL support

- Add URL detection and parsing utilities for GitHub URLs, local paths, and NPM packages
- Implement git operations for cloning and updating repositories with local caching
- Add automatic update checking mechanism for GitHub repositories
- Update provider-loader.ts to support multiple source types with comprehensive error handling
- Add comprehensive test coverage for all new functionality
- Include complete documentation with usage examples
- Support GitHub URLs: https://github.com/user/repo, user/repo@branch
- Support local paths: ./path, /absolute/path
- Support NPM packages: package-name, @scope/package
- Maintain backward compatibility with existing providers
- Add fallback mechanisms and interface validation
This commit is contained in:
Frostebite
2025-09-11 19:37:09 +01:00
parent 44bbd8c657
commit 8aa16937eb
9 changed files with 1688 additions and 56 deletions

484
dist/index.js generated vendored
View File

@@ -4436,6 +4436,269 @@ class LocalCloudRunner {
exports["default"] = LocalCloudRunner;
/***/ }),
/***/ 38562:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.ProviderGitManager = void 0;
const child_process_1 = __nccwpck_require__(32081);
const util_1 = __nccwpck_require__(73837);
const fs = __importStar(__nccwpck_require__(57147));
const path_1 = __importDefault(__nccwpck_require__(71017));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const provider_url_parser_1 = __nccwpck_require__(78799);
const execAsync = (0, util_1.promisify)(child_process_1.exec);
/**
* Manages git operations for provider repositories
*/
class ProviderGitManager {
/**
* Ensures the cache directory exists
*/
static ensureCacheDir() {
if (!fs.existsSync(this.CACHE_DIR)) {
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
cloud_runner_logger_1.default.log(`Created provider cache directory: ${this.CACHE_DIR}`);
}
}
/**
* Gets the local path for a cached repository
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
static getLocalPath(urlInfo) {
const cacheKey = (0, provider_url_parser_1.generateCacheKey)(urlInfo);
return path_1.default.join(this.CACHE_DIR, cacheKey);
}
/**
* Checks if a repository is already cloned locally
* @param urlInfo GitHub URL information
* @returns True if repository exists locally
*/
static isRepositoryCloned(urlInfo) {
const localPath = this.getLocalPath(urlInfo);
return fs.existsSync(localPath) && fs.existsSync(path_1.default.join(localPath, '.git'));
}
/**
* Clones a GitHub repository to the local cache
* @param urlInfo GitHub URL information
* @returns Clone result with success status and local path
*/
static async cloneRepository(urlInfo) {
this.ensureCacheDir();
const localPath = this.getLocalPath(urlInfo);
// Remove existing directory if it exists
if (fs.existsSync(localPath)) {
cloud_runner_logger_1.default.log(`Removing existing directory: ${localPath}`);
fs.rmSync(localPath, { recursive: true, force: true });
}
try {
cloud_runner_logger_1.default.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
cloud_runner_logger_1.default.log(`Executing: ${cloneCommand}`);
const { stderr } = await execAsync(cloneCommand, {
timeout: this.GIT_TIMEOUT,
cwd: this.CACHE_DIR,
});
if (stderr && !stderr.includes('warning')) {
cloud_runner_logger_1.default.log(`Git clone stderr: ${stderr}`);
}
cloud_runner_logger_1.default.log(`Successfully cloned repository to: ${localPath}`);
return {
success: true,
localPath,
};
}
catch (error) {
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
cloud_runner_logger_1.default.log(`Error: ${errorMessage}`);
return {
success: false,
localPath,
error: errorMessage,
};
}
}
/**
* Updates a locally cloned repository
* @param urlInfo GitHub URL information
* @returns Update result with success status and whether it was updated
*/
static async updateRepository(urlInfo) {
const localPath = this.getLocalPath(urlInfo);
if (!this.isRepositoryCloned(urlInfo)) {
return {
success: false,
updated: false,
error: 'Repository not found locally',
};
}
try {
cloud_runner_logger_1.default.log(`Updating repository: ${localPath}`);
// Fetch latest changes
await execAsync('git fetch origin', {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
// Check if there are updates
const { stdout: statusOutput } = await execAsync(`git status -uno`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
const hasUpdates = statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
if (hasUpdates) {
cloud_runner_logger_1.default.log(`Updates available, pulling latest changes...`);
// Reset to origin/branch to get latest changes
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
timeout: this.GIT_TIMEOUT,
cwd: localPath,
});
cloud_runner_logger_1.default.log(`Repository updated successfully`);
return {
success: true,
updated: true,
};
}
else {
cloud_runner_logger_1.default.log(`Repository is already up to date`);
return {
success: true,
updated: false,
};
}
}
catch (error) {
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
cloud_runner_logger_1.default.log(`Error: ${errorMessage}`);
return {
success: false,
updated: false,
error: errorMessage,
};
}
}
/**
* Ensures a repository is available locally (clone if needed, update if exists)
* @param urlInfo GitHub URL information
* @returns Local path to the repository
*/
static async ensureRepositoryAvailable(urlInfo) {
this.ensureCacheDir();
if (this.isRepositoryCloned(urlInfo)) {
cloud_runner_logger_1.default.log(`Repository already exists locally, checking for updates...`);
const updateResult = await this.updateRepository(urlInfo);
if (!updateResult.success) {
cloud_runner_logger_1.default.log(`Failed to update repository, attempting fresh clone...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
return this.getLocalPath(urlInfo);
}
else {
cloud_runner_logger_1.default.log(`Repository not found locally, cloning...`);
const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) {
throw new Error(`Failed to clone repository: ${cloneResult.error}`);
}
return cloneResult.localPath;
}
}
/**
* Gets the path to the provider module within a repository
* @param urlInfo GitHub URL information
* @param localPath Local path to the repository
* @returns Path to the provider module
*/
static getProviderModulePath(urlInfo, localPath) {
if (urlInfo.path) {
return path_1.default.join(localPath, urlInfo.path);
}
// Look for common provider entry points
const commonEntryPoints = [
'index.js',
'index.ts',
'src/index.js',
'src/index.ts',
'lib/index.js',
'lib/index.ts',
'dist/index.js',
'dist/index.js.map',
];
for (const entryPoint of commonEntryPoints) {
const fullPath = path_1.default.join(localPath, entryPoint);
if (fs.existsSync(fullPath)) {
cloud_runner_logger_1.default.log(`Found provider entry point: ${entryPoint}`);
return fullPath;
}
}
// Default to repository root
cloud_runner_logger_1.default.log(`No specific entry point found, using repository root`);
return localPath;
}
/**
* Cleans up old cached repositories (optional maintenance)
* @param maxAgeDays Maximum age in days for cached repositories
*/
static async cleanupOldRepositories(maxAgeDays = 30) {
this.ensureCacheDir();
try {
const entries = fs.readdirSync(this.CACHE_DIR, { withFileTypes: true });
const now = Date.now();
const maxAge = maxAgeDays * 24 * 60 * 60 * 1000; // Convert to milliseconds
for (const entry of entries) {
if (entry.isDirectory()) {
const entryPath = path_1.default.join(this.CACHE_DIR, entry.name);
const stats = fs.statSync(entryPath);
if (now - stats.mtime.getTime() > maxAge) {
cloud_runner_logger_1.default.log(`Cleaning up old repository: ${entry.name}`);
fs.rmSync(entryPath, { recursive: true, force: true });
}
}
}
}
catch (error) {
cloud_runner_logger_1.default.log(`Error during cleanup: ${error.message}`);
}
}
}
exports.ProviderGitManager = ProviderGitManager;
ProviderGitManager.CACHE_DIR = path_1.default.join(process.cwd(), '.provider-cache');
ProviderGitManager.GIT_TIMEOUT = 30000; // 30 seconds
/***/ }),
/***/ 45788:
@@ -4472,39 +4735,80 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.ProviderLoader = void 0;
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const provider_url_parser_1 = __nccwpck_require__(78799);
const provider_git_manager_1 = __nccwpck_require__(38562);
// import path from 'path'; // Not currently used
/**
* Dynamically load a provider package by name.
* @param providerName Name of the provider package to load
* Dynamically load a provider package by name, URL, or path.
* @param providerSource Provider source (name, URL, or path)
* @param buildParameters Build parameters passed to the provider constructor
* @throws Error when the provider cannot be loaded or does not implement ProviderInterface
*/
async function loadProvider(providerName, buildParameters) {
cloud_runner_logger_1.default.log(`Loading provider: ${providerName}`);
async function loadProvider(providerSource, buildParameters) {
cloud_runner_logger_1.default.log(`Loading provider: ${providerSource}`);
// Parse the provider source to determine its type
const sourceInfo = (0, provider_url_parser_1.parseProviderSource)(providerSource);
(0, provider_url_parser_1.logProviderSource)(providerSource, sourceInfo);
let modulePath;
let importedModule;
try {
// Map provider names to their module paths for built-in providers
const providerModuleMap = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
const modulePath = providerModuleMap[providerName] || providerName;
// Handle different source types
switch (sourceInfo.type) {
case 'github': {
cloud_runner_logger_1.default.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
// Ensure the repository is available locally
const localRepoPath = await provider_git_manager_1.ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
// Get the path to the provider module within the repository
modulePath = provider_git_manager_1.ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
cloud_runner_logger_1.default.log(`Loading provider from: ${modulePath}`);
break;
}
case 'local': {
modulePath = sourceInfo.path;
cloud_runner_logger_1.default.log(`Loading provider from local path: ${modulePath}`);
break;
}
case 'npm': {
modulePath = sourceInfo.packageName;
cloud_runner_logger_1.default.log(`Loading provider from NPM package: ${modulePath}`);
break;
}
default: {
// Fallback to built-in providers or direct import
const providerModuleMap = {
aws: './aws',
k8s: './k8s',
test: './test',
'local-docker': './docker',
'local-system': './local',
local: './local',
};
modulePath = providerModuleMap[providerSource] || providerSource;
cloud_runner_logger_1.default.log(`Loading provider from module path: ${modulePath}`);
break;
}
}
// Import the module
importedModule = await Promise.resolve().then(() => __importStar(require(modulePath)));
}
catch (error) {
throw new Error(`Failed to load provider package '${providerName}': ${error.message}`);
throw new Error(`Failed to load provider package '${providerSource}': ${error.message}`);
}
// Extract the provider class/function
const Provider = importedModule.default || importedModule;
// Validate that we have a constructor
if (typeof Provider !== 'function') {
throw new TypeError(`Provider package '${providerSource}' does not export a constructor function`);
}
// Instantiate the provider
let instance;
try {
instance = new Provider(buildParameters);
}
catch (error) {
throw new Error(`Failed to instantiate provider '${providerName}': ${error.message}`);
throw new Error(`Failed to instantiate provider '${providerSource}': ${error.message}`);
}
// Validate that the instance implements the required interface
const requiredMethods = [
'cleanupWorkflow',
'setupWorkflow',
@@ -4516,10 +4820,10 @@ async function loadProvider(providerName, buildParameters) {
];
for (const method of requiredMethods) {
if (typeof instance[method] !== 'function') {
throw new TypeError(`Provider package '${providerName}' does not implement ProviderInterface. Missing method '${method}'.`);
throw new TypeError(`Provider package '${providerSource}' does not implement ProviderInterface. Missing method '${method}'.`);
}
}
cloud_runner_logger_1.default.log(`Successfully loaded provider: ${providerName}`);
cloud_runner_logger_1.default.log(`Successfully loaded provider: ${providerSource}`);
return instance;
}
exports["default"] = loadProvider;
@@ -4528,14 +4832,14 @@ exports["default"] = loadProvider;
*/
class ProviderLoader {
/**
* Dynamically loads a provider by name (wrapper around loadProvider function)
* @param providerName - The name of the provider to load
* Dynamically loads a provider by name, URL, or path (wrapper around loadProvider function)
* @param providerSource - The provider source (name, URL, or path) to load
* @param buildParameters - Build parameters to pass to the provider constructor
* @returns Promise<ProviderInterface> - The loaded provider instance
* @throws Error if provider package is missing or doesn't implement ProviderInterface
*/
static async loadProvider(providerName, buildParameters) {
return loadProvider(providerName, buildParameters);
static async loadProvider(providerSource, buildParameters) {
return loadProvider(providerSource, buildParameters);
}
/**
* Gets a list of available provider names
@@ -4544,10 +4848,146 @@ class ProviderLoader {
static getAvailableProviders() {
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local'];
}
/**
* Cleans up old cached repositories
* @param maxAgeDays Maximum age in days for cached repositories (default: 30)
*/
static async cleanupCache(maxAgeDays = 30) {
await provider_git_manager_1.ProviderGitManager.cleanupOldRepositories(maxAgeDays);
}
/**
* Gets information about a provider source without loading it
* @param providerSource The provider source to analyze
* @returns ProviderSourceInfo object with parsed details
*/
static analyzeProviderSource(providerSource) {
return (0, provider_url_parser_1.parseProviderSource)(providerSource);
}
}
exports.ProviderLoader = ProviderLoader;
/***/ }),
/***/ 78799:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.logProviderSource = exports.isGitHubSource = exports.generateCacheKey = exports.parseProviderSource = void 0;
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
/**
* Parses a provider source string and determines its type and details
* @param source The provider source string (URL, path, or package name)
* @returns ProviderSourceInfo object with parsed details
*/
function parseProviderSource(source) {
// Check if it's a GitHub URL
const githubMatch = source.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
if (githubMatch) {
const [, owner, repo, branch, path] = githubMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: source,
};
}
// Check if it's a GitHub SSH URL
const githubSshMatch = source.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
if (githubSshMatch) {
const [, owner, repo, branch, path] = githubSshMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a shorthand GitHub reference (owner/repo)
const shorthandMatch = source.match(/^([^/@]+)\/([^/@]+)(?:@([^/]+))?(?:\/(.+))?$/);
if (shorthandMatch && !source.startsWith('.') && !source.startsWith('/') && !source.includes('\\')) {
const [, owner, repo, branch, path] = shorthandMatch;
return {
type: 'github',
owner,
repo,
branch: branch || 'main',
path: path || '',
url: `https://github.com/${owner}/${repo}`,
};
}
// Check if it's a local path
if (source.startsWith('./') || source.startsWith('../') || source.startsWith('/') || source.includes('\\')) {
return {
type: 'local',
path: source,
};
}
// Default to npm package
return {
type: 'npm',
packageName: source,
};
}
exports.parseProviderSource = parseProviderSource;
/**
* Generates a cache key for a GitHub repository
* @param urlInfo GitHub URL information
* @returns Cache key string
*/
function generateCacheKey(urlInfo) {
return `github_${urlInfo.owner}_${urlInfo.repo}_${urlInfo.branch}`.replace(/[^\w-]/g, '_');
}
exports.generateCacheKey = generateCacheKey;
/**
* Validates if a string looks like a valid GitHub URL or reference
* @param source The source string to validate
* @returns True if it looks like a GitHub reference
*/
function isGitHubSource(source) {
const parsed = parseProviderSource(source);
return parsed.type === 'github';
}
exports.isGitHubSource = isGitHubSource;
/**
* Logs the parsed provider source information
* @param source The original source string
* @param parsed The parsed source information
*/
function logProviderSource(source, parsed) {
cloud_runner_logger_1.default.log(`Provider source: ${source}`);
switch (parsed.type) {
case 'github':
cloud_runner_logger_1.default.log(` Type: GitHub repository`);
cloud_runner_logger_1.default.log(` Owner: ${parsed.owner}`);
cloud_runner_logger_1.default.log(` Repository: ${parsed.repo}`);
cloud_runner_logger_1.default.log(` Branch: ${parsed.branch}`);
if (parsed.path) {
cloud_runner_logger_1.default.log(` Path: ${parsed.path}`);
}
break;
case 'local':
cloud_runner_logger_1.default.log(` Type: Local path`);
cloud_runner_logger_1.default.log(` Path: ${parsed.path}`);
break;
case 'npm':
cloud_runner_logger_1.default.log(` Type: NPM package`);
cloud_runner_logger_1.default.log(` Package: ${parsed.packageName}`);
break;
}
}
exports.logProviderSource = logProviderSource;
/***/ }),
/***/ 63007:

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long