Files
unity-builder/src/model/cloud-runner/cloud-runner.ts
Frostebite a61c02481f Cloud Runner v0 - Reliable and trimmed down cloud runner (#353)
* Update cloud-runner-aws-pipeline.yml

* Update cloud-runner-k8s-pipeline.yml

* yarn build

* yarn build

* correct branch ref

* correct branch ref passed to target repo

* Create k8s-tests.yml

* Delete k8s-tests.yml

* correct branch ref passed to target repo

* correct branch ref passed to target repo

* Always describe AWS tasks for now, because unstable error handling

* Remove unused tree commands

* Use lfs guid sum

* Simple override cache push

* Simple override cache push and pull override to allow pure cloud storage driven caching

* Removal of early branch (breaks lfs caching)

* Remove unused tree commands

* Update action.yml

* Update action.yml

* Support cache and input override commands as input + full support custom hooks

* Increase k8s timeout

* replace filename being appended for unknclear reason

* cache key should not contain whitespaces

* Always try and deploy rook for k8s

* Apply k8s files for rook

* Update action.yml

* Apply k8s files for rook

* Apply k8s files for rook

* cache test and action description for kuber storage class

* Correct test and implement dependency health check and start

* GCP-secret run, cache key

* lfs smudge set explicit and undo explicit

* Run using external secret provider to speed up input

* Update cloud-runner-aws-pipeline.yml

* Add nodejs as build step dependency

* Add nodejs as build step dependency

* Cloud Runner Tests must be specified to capture logs from cloud runner for tests

* Cloud Runner Tests must be specified to capture logs from cloud runner for tests

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* better defaults for new inputs

* better defaults

* merge latest

* force build update

* use npm n to update node in unity builder

* use npm n to update node in unity builder

* use npm n to update node in unity builder

* correct new line

* quiet zipping

* quiet zipping

* default secrets for unity username and password

* default secrets for unity username and password

* ls active directory before lfs install

* Get cloud runner secrets from

* Get cloud runner secrets from

* Cleanup setup of default secrets

* Various fixes

* Cleanup setup of default secrets

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* AWS secrets manager support

* less caching logs

* default k8s storage class to pd-standard

* more readable build commands

* Capture aws exit code 1 reliably

* Always replace /head from branch

* k8s default storage class to standard-rwo

* cleanup

* further cleanup input

* further cleanup input

* further cleanup input

* further cleanup input

* further cleanup input

* folder sizes to inspect caching

* dir command for local cloud runner test

* k8s wait for pending because pvc will not create earlier

* prefer k8s standard storage

* handle empty string as cloud runner cluster input

* local-system is now used for cloud runner test implementation AND correctly unset test CLI input

* local-system is now used for cloud runner test implementation AND correctly unset test CLI input

* fix unterminated quote

* fix unterminated quote

* do not share build parameters in tests - in cloud runner this will cause conflicts with resouces of the same name

* remove head and heads from branch prefix

* fix reversed caching direction of cache-push

* fixes

* fixes

* fixes

* cachePull cli

* fixes

* fixes

* fixes

* fixes

* fixes

* order cache test to be first

* order cache test to be first

* fixes

* populate cache key instead of using branch

* cleanup cli

* garbage-collect-aws cli can iterate over aws resources and cli scans all ts files

* import cli methods

* import cli files explicitly

* import cli files explicitly

* import cli files explicitly

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* log parameters in cloud runner parameter test

* log parameters in cloud runner parameter test

* log parameters in cloud runner parameter test

* Cloud runner param test before caching because we have a fast local cache test now

* Using custom build path relative to repo root rather than project root

* aws-garbage-collect at end of pipeline

* aws-garbage-collect do not actually delete anything for now - just list

* remove some legacy du commands

* Update cloud-runner-aws-pipeline.yml

* log contents after cache pull and fix some scenarios with duplicate secrets

* log contents after cache pull and fix some scenarios with duplicate secrets

* log contents after cache pull and fix some scenarios with duplicate secrets

* PR comments

* Replace guid with uuid package

* use fileExists lambda instead of stat to check file exists in caching

* build failed results in core error message

* Delete sample.txt
2022-04-11 00:00:37 +01:00

92 lines
4.0 KiB
TypeScript

import AwsBuildPlatform from './providers/aws';
import { BuildParameters, Input } from '..';
import Kubernetes from './providers/k8s';
import CloudRunnerLogger from './services/cloud-runner-logger';
import { CloudRunnerStepState } from './cloud-runner-step-state';
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
import { CloudRunnerError } from './error/cloud-runner-error';
import { TaskParameterSerializer } from './services/task-parameter-serializer';
import * as core from '@actions/core';
import CloudRunnerSecret from './services/cloud-runner-secret';
import { ProviderInterface } from './providers/provider-interface';
import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
import TestCloudRunner from './providers/test';
import LocalCloudRunner from './providers/local';
import LocalDockerCloudRunner from './providers/local-docker';
class CloudRunner {
public static Provider: ProviderInterface;
static buildParameters: BuildParameters;
public static defaultSecrets: CloudRunnerSecret[];
public static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
private static setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup();
CloudRunner.buildParameters = buildParameters;
CloudRunner.setupBuildPlatform();
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
CloudRunner.cloudRunnerEnvironmentVariables = TaskParameterSerializer.readBuildEnvironmentVariables();
if (!buildParameters.isCliMode) {
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
}
for (const element of buildParameterPropertyNames) {
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
}
}
}
private static setupBuildPlatform() {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.cloudRunnerCluster}`);
switch (CloudRunner.buildParameters.cloudRunnerCluster) {
case 'k8s':
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
break;
case 'aws':
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
break;
case 'test':
CloudRunner.Provider = new TestCloudRunner();
break;
case 'local-system':
CloudRunner.Provider = new LocalCloudRunner();
break;
case 'local-docker':
CloudRunner.Provider = new LocalDockerCloudRunner();
break;
}
}
static async run(buildParameters: BuildParameters, baseImage: string) {
CloudRunner.setup(buildParameters);
try {
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
await CloudRunner.Provider.setup(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
const output = await new WorkflowCompositionRoot().run(
new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
);
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Cleanup shared cloud runner resources');
await CloudRunner.Provider.cleanup(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
CloudRunnerLogger.log(`Cleanup complete`);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
return output;
} catch (error) {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await CloudRunnerError.handleException(error);
throw error;
}
}
}
export default CloudRunner;