mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-01-28 19:49:08 +08:00
Cloud Runner Develop - 1.0 R.C 1 (#437)
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
@@ -1,5 +1,5 @@
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import { BuildParameters, Output } from '../../..';
|
||||
import { BuildParameters } from '../../..';
|
||||
import * as core from '@actions/core';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
@@ -7,39 +7,84 @@ import KubernetesStorage from './kubernetes-storage';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
||||
import KubernetesSecret from './kubernetes-secret';
|
||||
import waitUntil from 'async-wait-until';
|
||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import DependencyOverrideService from '../../services/depdency-override-service';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import KubernetesPods from './kubernetes-pods';
|
||||
|
||||
class Kubernetes implements ProviderInterface {
|
||||
private kubeConfig: k8s.KubeConfig;
|
||||
private kubeClient: k8s.CoreV1Api;
|
||||
private kubeClientBatch: k8s.BatchV1Api;
|
||||
private buildGuid: string = '';
|
||||
private buildParameters: BuildParameters;
|
||||
private pvcName: string = '';
|
||||
private secretName: string = '';
|
||||
private jobName: string = '';
|
||||
private namespace: string;
|
||||
private podName: string = '';
|
||||
private containerName: string = '';
|
||||
private cleanupCronJobName: string = '';
|
||||
private serviceAccountName: string = '';
|
||||
public static Instance: Kubernetes;
|
||||
public kubeConfig!: k8s.KubeConfig;
|
||||
public kubeClient!: k8s.CoreV1Api;
|
||||
public kubeClientBatch!: k8s.BatchV1Api;
|
||||
public buildGuid: string = '';
|
||||
public buildParameters!: BuildParameters;
|
||||
public pvcName: string = '';
|
||||
public secretName: string = '';
|
||||
public jobName: string = '';
|
||||
public namespace!: string;
|
||||
public podName: string = '';
|
||||
public containerName: string = '';
|
||||
public cleanupCronJobName: string = '';
|
||||
public serviceAccountName: string = '';
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
Kubernetes.Instance = this;
|
||||
this.kubeConfig = new k8s.KubeConfig();
|
||||
this.kubeConfig.loadFromDefault();
|
||||
this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api);
|
||||
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
||||
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||
|
||||
this.namespace = 'default';
|
||||
this.buildParameters = buildParameters;
|
||||
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||
}
|
||||
public async setup(
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
const pods = await this.kubeClient.listNamespacedPod(this.namespace);
|
||||
const serviceAccounts = await this.kubeClient.listNamespacedServiceAccount(this.namespace);
|
||||
const secrets = await this.kubeClient.listNamespacedSecret(this.namespace);
|
||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
||||
|
||||
return [
|
||||
...pods.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...serviceAccounts.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...secrets.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...jobs.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
];
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
public async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -48,12 +93,11 @@ class Kubernetes implements ProviderInterface {
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
try {
|
||||
this.pvcName = `unity-builder-pvc-${buildGuid}`;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${buildGuid}`;
|
||||
this.serviceAccountName = `service-account-${buildGuid}`;
|
||||
if (await DependencyOverrideService.CheckHealth()) {
|
||||
await DependencyOverrideService.TryStartDependencies();
|
||||
}
|
||||
this.buildParameters = buildParameters;
|
||||
const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
|
||||
this.pvcName = `unity-builder-pvc-${id}`;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
|
||||
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
|
||||
await KubernetesStorage.createPersistentVolumeClaim(
|
||||
buildParameters,
|
||||
this.pvcName,
|
||||
@@ -67,7 +111,7 @@ class Kubernetes implements ProviderInterface {
|
||||
}
|
||||
}
|
||||
|
||||
async runTask(
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
@@ -77,40 +121,22 @@ class Kubernetes implements ProviderInterface {
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
try {
|
||||
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
|
||||
|
||||
// Setup
|
||||
this.buildGuid = buildGuid;
|
||||
this.secretName = `build-credentials-${buildGuid}`;
|
||||
this.jobName = `unity-builder-job-${buildGuid}`;
|
||||
this.secretName = `build-credentials-${this.buildGuid}`;
|
||||
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
||||
this.containerName = `main`;
|
||||
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
||||
commands,
|
||||
image,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets,
|
||||
this.buildGuid,
|
||||
this.buildParameters,
|
||||
this.secretName,
|
||||
this.pvcName,
|
||||
this.jobName,
|
||||
k8s,
|
||||
);
|
||||
|
||||
// Run
|
||||
const jobResult = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||
CloudRunnerLogger.log(`Creating build job ${JSON.stringify(jobResult.body.metadata, undefined, 4)}`);
|
||||
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
CloudRunnerLogger.log('Job created');
|
||||
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||
this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
|
||||
CloudRunnerLogger.log('Watching pod until running');
|
||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||
let output = '';
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
try {
|
||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||
CloudRunnerLogger.log('Pod running, streaming logs');
|
||||
output = await KubernetesTaskRunner.runTask(
|
||||
this.kubeConfig,
|
||||
@@ -120,11 +146,25 @@ class Kubernetes implements ProviderInterface {
|
||||
'main',
|
||||
this.namespace,
|
||||
);
|
||||
break;
|
||||
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
|
||||
|
||||
if (!running) {
|
||||
CloudRunnerLogger.log(`Pod not found, assumed ended!`);
|
||||
break;
|
||||
} else {
|
||||
CloudRunnerLogger.log('Pod still running, recovering stream...');
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.message.includes(`HTTP`)) {
|
||||
const reason = error.response?.body?.reason || error.reason || ``;
|
||||
const errorMessage = error.message || ``;
|
||||
|
||||
const continueStreaming = reason === `NotFound` || errorMessage.includes(`dial timeout, backstop`);
|
||||
if (continueStreaming) {
|
||||
CloudRunnerLogger.log('Log Stream Container Not Found');
|
||||
await new Promise((resolve) => resolve(5000));
|
||||
continue;
|
||||
} else {
|
||||
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -140,6 +180,44 @@ class Kubernetes implements ProviderInterface {
|
||||
}
|
||||
}
|
||||
|
||||
private async createNamespacedJob(
|
||||
commands: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
) {
|
||||
for (let index = 0; index < 3; index++) {
|
||||
try {
|
||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
||||
commands,
|
||||
image,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets,
|
||||
this.buildGuid,
|
||||
this.buildParameters,
|
||||
this.secretName,
|
||||
this.pvcName,
|
||||
this.jobName,
|
||||
k8s,
|
||||
);
|
||||
await new Promise((promise) => setTimeout(promise, 15000));
|
||||
await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||
CloudRunnerLogger.log(`Build job created`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
CloudRunnerLogger.log('Job created');
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setPodNameAndContainerName(pod: k8s.V1Pod) {
|
||||
this.podName = pod.metadata?.name || '';
|
||||
this.containerName = pod.status?.containerStatuses?.[0].name || '';
|
||||
@@ -150,32 +228,24 @@ class Kubernetes implements ProviderInterface {
|
||||
try {
|
||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log('Failed to cleanup, error:');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
CloudRunnerLogger.log('Abandoning cleanup, build error:');
|
||||
throw error;
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Failed to cleanup`);
|
||||
if (error.response.body.reason !== `NotFound`) {
|
||||
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const jobBody = (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
|
||||
const podBody = (await this.kubeClient.readNamespacedPod(this.podName, this.namespace)).body;
|
||||
|
||||
return (jobBody === null || jobBody.status?.active === 0) && podBody === null;
|
||||
},
|
||||
{
|
||||
timeout: 500000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch {}
|
||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Failed to cleanup secret`);
|
||||
CloudRunnerLogger.log(error.response.body.reason);
|
||||
}
|
||||
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
|
||||
CloudRunnerLogger.log('cleaning up finished');
|
||||
}
|
||||
|
||||
async cleanup(
|
||||
async cleanupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -183,11 +253,19 @@ class Kubernetes implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
if (buildParameters.retainWorkspace) {
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`deleting PVC`);
|
||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||
await Output.setBuildVersion(buildParameters.buildVersion);
|
||||
// eslint-disable-next-line unicorn/no-process-exit
|
||||
process.exit();
|
||||
|
||||
try {
|
||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
||||
CloudRunnerLogger.log('cleaned up PVC and Service Account');
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {
|
||||
|
||||
Reference in New Issue
Block a user