actions/unity-builder
2
0
Fork 0
mirror of https://github.com/game-ci/unity-builder.git synced 2026-02-05 17:29:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cloud runner develop - latest fixes (#524)

Browse Source 
Cloud runner develop - latest fixes (#524)
This commit is contained in:
Frostebite
2023-03-27 12:14:23 +01:00
committed by GitHub
parent 309d668d63
commit 7abb3a409d
91 changed files with 4943 additions and 3599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/model/cloud-runner/providers/aws/aws-base-stack.ts
View File

@@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import * as SDK from 'aws-sdk';
import { BaseStackFormation } from './cloud-formations/base-stack-formation';

2
src/model/cloud-runner/providers/aws/aws-error.ts
View File

@@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as SDK from 'aws-sdk';
import * as core from '@actions/core';
import CloudRunner from '../../cloud-runner';

25
src/model/cloud-runner/providers/aws/aws-job-stack.ts
View File

@@ -1,12 +1,12 @@
import * as SDK from 'aws-sdk';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSError } from './aws-error';
import CloudRunner from '../../cloud-runner';
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
import CloudRunnerOptions from '../../cloud-runner-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
export class AWSJobStack {
@@ -27,21 +27,19 @@ export class AWSJobStack {
): Promise<CloudRunnerAWSTaskDef> {
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024';
const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072';
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerCpu:
Default: 1024`,
`ContainerCpu:
Default: ${Number.parseInt(cpu)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
);
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerMemory:
Default: 2048`,
`ContainerMemory:
Default: ${Number.parseInt(memory)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
);
if (CloudRunnerOptions.watchCloudRunnerToEnd) {
if (!CloudRunnerOptions.asyncCloudRunner) {
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'# template resources logstream',
@@ -116,7 +114,7 @@ export class AWSJobStack {
...secretsMappedToCloudFormationParameters,
];
CloudRunnerLogger.log(
`Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`,
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
);
let previousStackExists = true;
while (previousStackExists) {
@@ -140,11 +138,16 @@ export class AWSJobStack {
Capabilities: ['CAPABILITY_IAM'],
Parameters: parameters,
};
try {
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
await CF.createStack(createStackInput).promise();
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
for (const parameter of parameters) {
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
}
}
} catch (error) {
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
throw error;
@@ -180,7 +183,7 @@ export class AWSJobStack {
if (CloudRunnerOptions.useCleanupCron) {
try {
CloudRunnerLogger.log(`Creating job cleanup formation`);
CF.createStack(createCleanupStackInput).promise();
await CF.createStack(createCleanupStackInput).promise();
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
} catch (error) {

66
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@@ -1,14 +1,14 @@
import * as AWS from 'aws-sdk';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import * as core from '@actions/core';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import * as zlib from 'node:zlib';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { Input } from '../../..';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
import CloudRunnerOptions from '../../cloud-runner-options';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import GitHub from '../../../github';
class AWSTaskRunner {
@@ -32,7 +32,7 @@ class AWSTaskRunner {
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
const task = await AWSTaskRunner.ECS.runTask({
const runParameters = {
cluster,
taskDefinition,
platformVersion: '1.4.0',
@@ -41,7 +41,7 @@ class AWSTaskRunner {
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},
@@ -53,16 +53,23 @@ class AWSTaskRunner {
securityGroups: [ContainerSecurityGroup],
},
},
}).promise();
};
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
throw new Error(`Container Overrides length must be at most 8192`);
}
const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
CloudRunnerLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
CloudRunnerOptions.watchCloudRunnerToEnd
} Async:${CloudRunnerOptions.asyncCloudRunner}`,
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
CloudRunnerOptions.asyncCloudRunner
}`,
);
if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
if (CloudRunnerOptions.asyncCloudRunner) {
const shouldCleanup: boolean = false;
const output: string = '';
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
@@ -72,26 +79,31 @@ class AWSTaskRunner {
CloudRunnerLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
await new Promise((resolve) => resolve(5000));
const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
const containerState = taskData.containers?.[0];
const exitCode = containerState?.exitCode || undefined;
let exitCode;
let containerState;
let taskData;
while (exitCode === undefined) {
await new Promise((resolve) => resolve(10000));
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
containerState = taskData.containers?.[0];
exitCode = containerState?.exitCode;
}
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
if (exitCode === undefined) {
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
}
const wasSuccessful = exitCode === 0;
if (wasSuccessful) {
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
return { output, shouldCleanup };
} else {
if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
throw new Error('Container exited with code 1');
}
const message = `Cloud runner job exit code ${exitCode}`;
taskData.overrides = undefined;
taskData.attachments = undefined;
CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
throw new Error(message);
}
if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
throw new Error('Container exited with code 1');
}
throw new Error(`Task failed`);
}
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
@@ -129,7 +141,7 @@ class AWSTaskRunner {
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
let shouldReadLogs = true;

2
src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
View File

@@ -30,7 +30,7 @@ Parameters:
Type: Number
Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory:
Default: 2048
Default: 4096
Type: Number
Description: How much memory in megabytes to give the container
BUILDGUID:

21
src/model/cloud-runner/providers/aws/index.ts
View File

@@ -1,11 +1,11 @@
import * as SDK from 'aws-sdk';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
@@ -13,13 +13,13 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../cloud-runner-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
this.baseStackName = buildParameters.awsStackName;
}
async listResources(): Promise<ProviderResource[]> {
await TaskService.getCloudFormationJobStacks();
@@ -75,7 +75,11 @@ class AWSBuildEnvironment implements ProviderInterface {
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
) {
process.env.AWS_REGION = Input.region;
const CF = new SDK.CloudFormation();
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
}
async runTaskInWorkflow(
buildGuid: string,
@@ -94,8 +98,6 @@ class AWSBuildEnvironment implements ProviderInterface {
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
CF,
buildGuid,
@@ -143,6 +145,9 @@ class AWSBuildEnvironment implements ProviderInterface {
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackName,
}).promise();
await CF.waitFor('stackDeleteComplete', {
StackName: `${taskDef.taskDefStackName}-cleanup`,
}).promise();
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete');
}

2
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts
View File

@@ -1,6 +1,6 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { TaskService } from './task-service';
export class GarbageCollectionService {

4
src/model/cloud-runner/providers/aws/services/task-service.ts
View File

@@ -1,6 +1,6 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import { ListObjectsRequest } from 'aws-sdk/clients/s3';
@@ -161,7 +161,7 @@ export class TaskService {
process.env.AWS_REGION = Input.region;
const s3 = new AWS.S3();
const listRequest: ListObjectsRequest = {
Bucket: CloudRunner.buildParameters.awsBaseStackName,
Bucket: CloudRunner.buildParameters.awsStackName,
};
const results = await s3.listObjects(listRequest).promise();

22
src/model/cloud-runner/providers/docker/index.ts
View File

@@ -1,20 +1,21 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import Docker from '../../../docker';
import { Action } from '../../..';
import { writeFileSync } from 'fs';
import { writeFileSync } from 'node:fs';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
import fs from 'node:fs';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters: BuildParameters | undefined;
public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> {
return new Promise((resolve) => resolve([]));
@@ -51,14 +52,14 @@ class LocalDockerCloudRunner implements ProviderInterface {
if (
fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
)
) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
@@ -118,7 +119,7 @@ set -e
mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
${commands}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
`;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
@@ -149,6 +150,7 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
},
},
true,
false,
);
return myOutput;

153
src/model/cloud-runner/providers/k8s/index.ts
View File

@@ -2,19 +2,18 @@ import * as k8s from '@kubernetes/client-node';
import { BuildParameters } from '../../..';
import * as core from '@actions/core';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import KubernetesStorage from './kubernetes-storage';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import KubernetesTaskRunner from './kubernetes-task-runner';
import KubernetesSecret from './kubernetes-secret';
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
import KubernetesServiceAccount from './kubernetes-service-account';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import KubernetesPods from './kubernetes-pods';
class Kubernetes implements ProviderInterface {
public static Instance: Kubernetes;
@@ -94,16 +93,8 @@ class Kubernetes implements ProviderInterface {
) {
try {
this.buildParameters = buildParameters;
const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
await KubernetesStorage.createPersistentVolumeClaim(
buildParameters,
this.pvcName,
this.kubeClient,
this.namespace,
);
await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
} catch (error) {
@@ -124,74 +115,99 @@ class Kubernetes implements ProviderInterface {
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
// Setup
const id = BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
? CloudRunner.lockedWorkspace
: this.buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
await KubernetesStorage.createPersistentVolumeClaim(
this.buildParameters,
this.pvcName,
this.kubeClient,
this.namespace,
);
this.buildGuid = buildGuid;
this.secretName = `build-credentials-${this.buildGuid}`;
this.jobName = `unity-builder-job-${this.buildGuid}`;
this.containerName = `main`;
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
let output = '';
// eslint-disable-next-line no-constant-condition
while (true) {
try {
CloudRunnerLogger.log('Pod running, streaming logs');
output = await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
this.jobName,
this.podName,
'main',
this.namespace,
);
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
try {
CloudRunnerLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
if (!running) {
CloudRunnerLogger.log(`Pod not found, assumed ended!`);
break;
} else {
CloudRunnerLogger.log('Pod still running, recovering stream...');
}
await this.cleanupTaskResources();
} catch (error: any) {
let errorParsed;
try {
errorParsed = JSON.parse(error);
} catch {
errorParsed = error;
}
const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
const errorMessage = errorParsed.message || reason;
const continueStreaming =
errorMessage.includes(`dial timeout, backstop`) ||
errorMessage.includes(`HttpError: HTTP request failed`) ||
errorMessage.includes(`an error occurred when try to find container`) ||
errorMessage.includes(`not found`) ||
errorMessage.includes(`Not Found`);
if (continueStreaming) {
CloudRunnerLogger.log('Log Stream Container Not Found');
await new Promise((resolve) => resolve(5000));
continue;
} else {
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
throw error;
}
}
CloudRunnerLogger.log('Pod running, streaming logs');
CloudRunnerLogger.log(
`Starting logs follow for pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace} pvc: ${this.pvcName} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
);
output += await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
this.jobName,
this.podName,
this.containerName,
this.namespace,
);
} catch (error: any) {
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
JSON.stringify(
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(this.podName)),
undefined,
4,
),
);
await this.cleanupTaskResources();
throw error;
}
await this.cleanupTaskResources();
return output;
} catch (error) {
CloudRunnerLogger.log('Running job failed');
core.error(JSON.stringify(error, undefined, 4));
await this.cleanupTaskResources();
// await this.cleanupTaskResources();
throw error;
}
}
private async createJob(
commands: string,
image: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
}
private async doesJobExist(name: string) {
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
return jobs.body.items.some((x) => x.metadata?.name === name);
}
private async doesFailedJobExist() {
const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
return podStatus.body.status?.phase === `Failed`;
}
private async createNamespacedJob(
commands: string,
image: string,
@@ -215,14 +231,15 @@ class Kubernetes implements ProviderInterface {
this.pvcName,
this.jobName,
k8s,
this.containerName,
);
await new Promise((promise) => setTimeout(promise, 15000));
await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
CloudRunnerLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
CloudRunnerLogger.log('Job created');
return;
return result.body.metadata?.name;
} catch (error) {
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
throw error;
@@ -232,7 +249,7 @@ class Kubernetes implements ProviderInterface {
setPodNameAndContainerName(pod: k8s.V1Pod) {
this.podName = pod.metadata?.name || '';
this.containerName = pod.status?.containerStatuses?.[0].name || '';
this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
}
async cleanupTaskResources() {
@@ -265,7 +282,7 @@ class Kubernetes implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
if (buildParameters.retainWorkspace) {
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
return;
}
CloudRunnerLogger.log(`deleting PVC`);

80
src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
View File

@@ -1,8 +1,8 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunner from '../../cloud-runner';
class KubernetesJobSpecFactory {
@@ -19,63 +19,8 @@ class KubernetesJobSpecFactory {
pvcName: string,
jobName: string,
k8s: any,
containerName: string,
) {
environment.push(
...[
{
name: 'GITHUB_SHA',
value: buildGuid,
},
{
name: 'GITHUB_WORKSPACE',
value: '/data/repo',
},
{
name: 'PROJECT_PATH',
value: buildParameters.projectPath,
},
{
name: 'BUILD_PATH',
value: buildParameters.buildPath,
},
{
name: 'BUILD_FILE',
value: buildParameters.buildFile,
},
{
name: 'BUILD_NAME',
value: buildParameters.buildName,
},
{
name: 'BUILD_METHOD',
value: buildParameters.buildMethod,
},
{
name: 'CUSTOM_PARAMETERS',
value: buildParameters.customParameters,
},
{
name: 'CHOWN_FILES_TO',
value: buildParameters.chownFilesTo,
},
{
name: 'BUILD_TARGET',
value: buildParameters.targetPlatform,
},
{
name: 'ANDROID_VERSION_CODE',
value: buildParameters.androidVersionCode.toString(),
},
{
name: 'ANDROID_KEYSTORE_NAME',
value: buildParameters.androidKeystoreName,
},
{
name: 'ANDROID_KEYALIAS_NAME',
value: buildParameters.androidKeyaliasName,
},
],
);
const job = new k8s.V1Job();
job.apiVersion = 'batch/v1';
job.kind = 'Job';
@@ -87,6 +32,7 @@ class KubernetesJobSpecFactory {
},
};
job.spec = {
ttlSecondsAfterFinished: 9999,
backoffLimit: 0,
template: {
spec: {
@@ -100,16 +46,20 @@ class KubernetesJobSpecFactory {
],
containers: [
{
name: 'main',
ttlSecondsAfterFinished: 9999,
name: containerName,
image,
command: ['/bin/sh'],
args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
args: [
'-c',
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
],
workingDir: `${workingDirectory}`,
resources: {
requests: {
memory: buildParameters.cloudRunnerMemory || '750M',
cpu: buildParameters.cloudRunnerCpu || '1',
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
},
},
env: [
@@ -135,7 +85,7 @@ class KubernetesJobSpecFactory {
volumeMounts: [
{
name: 'build-mount',
mountPath: `/${mountdir}`,
mountPath: `${mountdir}`,
},
],
lifecycle: {
@@ -158,7 +108,7 @@ class KubernetesJobSpecFactory {
},
};
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
return job;
}

8
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
View File

@@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
class KubernetesPods {
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
@@ -12,6 +12,12 @@ class KubernetesPods {
return running;
}
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
const phase = pods?.status?.phase || 'undefined status';
return phase;
}
}
export default KubernetesPods;

4
src/model/cloud-runner/providers/k8s/kubernetes-secret.ts
View File

@@ -1,7 +1,7 @@
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import * as k8s from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as base64 from 'base-64';
class KubernetesSecret {

2
src/model/cloud-runner/providers/k8s/kubernetes-storage.ts
View File

@@ -2,7 +2,7 @@ import waitUntil from 'async-wait-until';
import * as core from '@actions/core';
import * as k8s from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { IncomingMessage } from 'node:http';
import GitHub from '../../../github';

168
src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts
View File

@@ -1,12 +1,15 @@
import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
import { Writable } from 'stream';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import * as core from '@actions/core';
import { CloudRunnerStatics } from '../../cloud-runner-statics';
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import waitUntil from 'async-wait-until';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunner from '../../cloud-runner';
import KubernetesPods from './kubernetes-pods';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
class KubernetesTaskRunner {
static lastReceivedTimestamp: number = 0;
static readonly maxRetry: number = 3;
static lastReceivedMessage: string = ``;
static async runTask(
kubeConfig: KubeConfig,
kubeClient: CoreV1Api,
@@ -15,84 +18,120 @@ class KubernetesTaskRunner {
containerName: string,
namespace: string,
) {
CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
const stream = new Writable();
let output = '';
let didStreamAnyLogs: boolean = false;
let shouldReadLogs = true;
let shouldCleanup = true;
stream._write = (chunk, encoding, next) => {
didStreamAnyLogs = true;
let message = chunk.toString().trimRight(`\n`);
message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
message,
shouldReadLogs,
shouldCleanup,
output,
));
next();
};
const logOptions = {
follow: true,
pretty: false,
previous: false,
};
try {
const resultError = await new Promise((resolve) =>
new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
let sinceTime = ``;
let retriesAfterFinish = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const lastReceivedMessage =
KubernetesTaskRunner.lastReceivedTimestamp > 0
? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
: ``;
CloudRunnerLogger.log(
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
);
stream.destroy();
if (resultError) {
throw resultError;
if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
const dateTimeIsoString = currentDate.toISOString();
sinceTime = ` --since-time="${dateTimeIsoString}"`;
}
if (!didStreamAnyLogs) {
core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
core.error(
JSON.stringify(
{
events: (await kubeClient.listNamespacedEvent(namespace)).body.items
.filter((x) => {
return x.involvedObject.name === podName || x.involvedObject.name === jobName;
})
.map((x) => {
return {
type: x.involvedObject.kind,
name: x.involvedObject.name,
message: x.message,
};
}),
},
undefined,
4,
),
let extraFlags = ``;
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
? ` -f -c ${containerName}`
: ` --previous`;
let lastMessageSeenIncludedInChunk = false;
let lastMessageSeen = false;
let logs;
try {
logs = await CloudRunnerSystem.Run(
`kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
false,
true,
);
throw new Error(`No logs streamed from k8s`);
} catch (error: any) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
if (continueStreaming) {
continue;
}
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
retriesAfterFinish++;
continue;
}
throw error;
}
} catch (error) {
if (stream) {
stream.destroy();
const splitLogs = logs.split(`\n`);
for (const chunk of splitLogs) {
if (
chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
) {
CloudRunnerLogger.log(`Previous log message found ${chunk}`);
lastMessageSeenIncludedInChunk = true;
}
}
for (const chunk of splitLogs) {
const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
lastMessageSeen = true;
}
if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
continue;
}
const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
KubernetesTaskRunner.lastReceivedMessage = chunk;
KubernetesTaskRunner.lastReceivedTimestamp = newDate;
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
message,
shouldReadLogs,
shouldCleanup,
output,
));
}
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream');
break;
}
throw error;
}
CloudRunnerLogger.log('end of log stream');
return output;
}
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
let success: boolean = false;
let message = ``;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
await waitUntil(
async () => {
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
const phase = status?.body.status?.phase;
success = phase === 'Running';
CloudRunnerLogger.log(
`${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
status.body.status?.conditions?.[0].message || ''
}`,
);
message = `Phase:${status.body.status?.phase} \n Reason:${
status.body.status?.conditions?.[0].reason || ''
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
// CloudRunnerLogger.log(
// JSON.stringify(
// (await kubeClient.listNamespacedEvent(namespace)).body.items
// .map((x) => {
// return {
// message: x.message || ``,
// name: x.metadata.name || ``,
// reason: x.reason || ``,
// };
// })
// .filter((x) => x.name.includes(podName)),
// undefined,
// 4,
// ),
// );
if (success || phase !== 'Pending') return true;
return false;
@@ -102,6 +141,9 @@ class KubernetesTaskRunner {
intervalBetweenAttempts: 15000,
},
);
if (!success) {
CloudRunnerLogger.log(message);
}
return success;
}

8
src/model/cloud-runner/providers/local/index.ts
View File

@@ -1,9 +1,9 @@
import BuildParameters from '../../../build-parameters';
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';

4
src/model/cloud-runner/providers/provider-interface.ts
View File

@@ -1,6 +1,6 @@
import BuildParameters from '../../build-parameters';
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import { ProviderResource } from './provider-resource';
import { ProviderWorkflow } from './provider-workflow';

6
src/model/cloud-runner/providers/test/index.ts
View File

@@ -1,8 +1,8 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';