Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)

* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
2026-02-04 08:09:08 +08:00 · 2023-01-20 17:40:57 +00:00
parent a45155c578
commit e334dc785a
39 changed files with 3083 additions and 399 deletions
--- a/src/model/cloud-runner/services/cloud-runner-custom-hooks.ts
+++ b/src/model/cloud-runner/services/cloud-runner-custom-hooks.ts
@@ -5,7 +5,8 @@ import { RemoteClientLogger } from '../remote-client/remote-client-logger';
 import path from 'path';
 import CloudRunnerOptions from '../cloud-runner-options';
 import * as fs from 'fs';
-import CloudRunnerLogger from './cloud-runner-logger';
+
+// import CloudRunnerLogger from './cloud-runner-logger';

 export class CloudRunnerCustomHooks {
  // TODO also accept hooks as yaml files in the repo
@@ -83,7 +84,7 @@ export class CloudRunnerCustomHooks {

    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {

-    CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
+    // CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);

    // }
    const isArray = steps.replace(/\s/g, ``)[0] === `-`;
--- a/src/model/cloud-runner/services/cloud-runner-custom-steps.ts
+++ b/src/model/cloud-runner/services/cloud-runner-custom-steps.ts
@@ -1,5 +1,4 @@
 import YAML from 'yaml';
-import CloudRunnerSecret from './cloud-runner-secret';
 import CloudRunner from '../cloud-runner';
 import * as core from '@actions/core';
 import { CustomWorkflow } from '../workflows/custom-workflow';
@@ -9,6 +8,7 @@ import * as fs from 'fs';
 import Input from '../../input';
 import CloudRunnerOptions from '../cloud-runner-options';
 import CloudRunnerLogger from './cloud-runner-logger';
+import { CustomStep } from './custom-step';

 export class CloudRunnerCustomSteps {
  static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
@@ -43,8 +43,14 @@ export class CloudRunnerCustomSteps {
    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
    aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4 s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4
-    rm /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4
+    aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
+        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+      } s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
+        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+      }
+    rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
+        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+      }
  secrets:
  - name: awsAccessKeyId
    value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@@ -52,6 +58,62 @@ export class CloudRunnerCustomSteps {
    value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
  - name: awsDefaultRegion
    value: ${process.env.AWS_REGION || ``}
+- name: aws-s3-pull-build
+  image: amazon/aws-cli
+  commands: |
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
+    aws configure set region $AWS_DEFAULT_REGION --profile default
+    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
+    aws s3 cp s3://${
+      CloudRunner.buildParameters.awsBaseStackName
+    }/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
+        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+      } /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
+        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+      }
+  secrets:
+    - name: awsAccessKeyId
+    - name: awsSecretAccessKey
+    - name: awsDefaultRegion
+    - name: BUILD_GUID_TARGET
+- name: steam-deploy-client
+  image: steamcmd/steamcmd
+  commands: |
+    apt-get update
+    apt-get install -y curl tar coreutils git tree > /dev/null
+    curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
+  secrets:
+    - name: STEAM_USERNAME
+    - name: STEAM_PASSWORD
+    - name: STEAM_APPID
+    - name: STEAM_SSFN_FILE_NAME
+    - name: STEAM_SSFN_FILE_CONTENTS
+    - name: STEAM_CONFIG_VDF_1
+    - name: STEAM_CONFIG_VDF_2
+    - name: STEAM_CONFIG_VDF_3
+    - name: STEAM_CONFIG_VDF_4
+    - name: BUILD_GUID_TARGET
+    - name: RELEASE_BRANCH
+- name: steam-deploy-project
+  image: steamcmd/steamcmd
+  commands: |
+    apt-get update
+    apt-get install -y curl tar coreutils git tree > /dev/null
+    curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
+  secrets:
+    - name: STEAM_USERNAME
+    - name: STEAM_PASSWORD
+    - name: STEAM_APPID
+    - name: STEAM_SSFN_FILE_NAME
+    - name: STEAM_SSFN_FILE_CONTENTS
+    - name: STEAM_CONFIG_VDF_1
+    - name: STEAM_CONFIG_VDF_2
+    - name: STEAM_CONFIG_VDF_3
+    - name: STEAM_CONFIG_VDF_4
+    - name: BUILD_GUID_2
+    - name: RELEASE_BRANCH
 - name: aws-s3-upload-cache
  image: amazon/aws-cli
  hook: after
@@ -59,9 +121,13 @@ export class CloudRunnerCustomSteps {
    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
    aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs
+    aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
+      CloudRunner.buildParameters.awsBaseStackName
+    }/cloud-runner-cache/$CACHE_KEY/lfs
    rm -r /data/cache/$CACHE_KEY/lfs
-    aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library
+    aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
+      CloudRunner.buildParameters.awsBaseStackName
+    }/cloud-runner-cache/$CACHE_KEY/Library
    rm -r /data/cache/$CACHE_KEY/Library
  secrets:
  - name: awsAccessKeyId
@@ -77,13 +143,13 @@ export class CloudRunnerCustomSteps {
    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
    aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
-    aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/ || true
-    BUCKET1="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library/"
+    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
+    BUCKET1="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
    aws s3 ls $BUCKET1 || true
    OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
    aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
-    BUCKET2="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs/"
+    BUCKET2="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
    aws s3 ls $BUCKET2 || true
    OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
    aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
@@ -135,21 +201,21 @@ export class CloudRunnerCustomSteps {
    if (steps === '') {
      return [];
    }
-
-    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
-
-    // CloudRunnerLogger.log(`Parsing build steps: ${steps}`);
-
-    // }
    const isArray = steps.replace(/\s/g, ``)[0] === `-`;
-    if (CloudRunner.buildParameters?.cloudRunnerDebug) {
-      CloudRunnerLogger.log(`Parsing: ${steps}`);
-    }
    const object: CustomStep[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
    for (const step of object) {
      CloudRunnerCustomSteps.ConvertYamlSecrets(step);
      if (step.secrets === undefined) {
        step.secrets = [];
+      } else {
+        for (const secret of step.secrets) {
+          if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
+            if (CloudRunner.buildParameters?.cloudRunnerDebug) {
+              CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
+            }
+            secret.ParameterValue = process.env[secret.ParameterKey] || ``;
+          }
+        }
      }
      if (step.image === undefined) {
        step.image = `ubuntu`;
@@ -201,10 +267,3 @@ export class CloudRunnerCustomSteps {
    return output;
  }
 }
-export class CustomStep {
-  public commands;
-  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
-  public name;
-  public image: string = `ubuntu`;
-  public hook!: string;
-}
--- a/src/model/cloud-runner/services/custom-step.ts
+++ b/src/model/cloud-runner/services/custom-step.ts
@@ -0,0 +1,9 @@
+import CloudRunnerSecret from './cloud-runner-secret';
+
+export class CustomStep {
+  public commands;
+  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
+  public name;
+  public image: string = `ubuntu`;
+  public hook!: string;
+}
--- a/src/model/cloud-runner/services/follow-log-stream-service.ts
+++ b/src/model/cloud-runner/services/follow-log-stream-service.ts
@@ -2,6 +2,7 @@ import CloudRunnerLogger from './cloud-runner-logger';
 import * as core from '@actions/core';
 import CloudRunner from '../cloud-runner';
 import { CloudRunnerStatics } from '../cloud-runner-statics';
+import GitHub from '../../github';

 export class FollowLogStreamService {
  public static handleIteration(message, shouldReadLogs, shouldCleanup, output) {
@@ -9,11 +10,14 @@ export class FollowLogStreamService {
      CloudRunnerLogger.log('End of log transmission received');
      shouldReadLogs = false;
    } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
+      GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
      core.warning('LIBRARY NOT FOUND!');
      core.setOutput('library-found', 'false');
    } else if (message.includes('Build succeeded')) {
+      GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
      core.setOutput('build-result', 'success');
    } else if (message.includes('Build fail')) {
+      GitHub.updateGitHubCheck(`Build failed`, `Build failed`);
      core.setOutput('build-result', 'failed');
      core.setFailed('unity build failed');
      core.error('BUILD FAILED!');
--- a/src/model/cloud-runner/services/shared-workspace-locking.ts
+++ b/src/model/cloud-runner/services/shared-workspace-locking.ts
@@ -5,8 +5,12 @@ import CloudRunnerOptions from '../cloud-runner-options';
 import BuildParameters from '../../build-parameters';
 import CloudRunner from '../cloud-runner';
 export class SharedWorkspaceLocking {
-  private static readonly workspaceBucketRoot = `s3://game-ci-test-storage/`;
-  private static readonly workspaceRoot = `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
+  private static get workspaceBucketRoot() {
+    return `s3://${CloudRunner.buildParameters.awsBaseStackName}/`;
+  }
+  private static get workspaceRoot() {
+    return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
+  }
  public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
    if (!(await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext))) {
      return [];
@@ -19,14 +23,11 @@ export class SharedWorkspaceLocking {
    ).map((x) => x.replace(`/`, ``));
  }
  public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
-    return (
-      (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`))
-        .map((x) => x.replace(`/`, ``))
-        .includes(`locks`) &&
-      (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`))
-        .map((x) => x.replace(`/`, ``))
-        .includes(buildParametersContext.cacheKey)
-    );
+    await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`);
+
+    return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`))
+      .map((x) => x.replace(`/`, ``))
+      .includes(buildParametersContext.cacheKey);
  }
  public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
    if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
@@ -49,20 +50,27 @@ export class SharedWorkspaceLocking {
    if (!CloudRunnerOptions.retainWorkspaces) {
      return;
    }
-    if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
-      const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
-      CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
-      for (const element of workspaces) {
-        await new Promise((promise) => setTimeout(promise, 1000));
-        const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
-        CloudRunnerLogger.log(`run agent: ${runId} try lock workspace: ${element} result: ${lockResult}`);

-        if (lockResult) {
-          CloudRunner.lockedWorkspace = element;
+    try {
+      if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
+        const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
+        CloudRunnerLogger.log(
+          `run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`,
+        );
+        for (const element of workspaces) {
+          await new Promise((promise) => setTimeout(promise, 1000));
+          const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
+          CloudRunnerLogger.log(`run agent: ${runId} try lock workspace: ${element} result: ${lockResult}`);

-          return true;
+          if (lockResult) {
+            CloudRunner.lockedWorkspace = element;
+
+            return true;
+          }
        }
      }
+    } catch {
+      return;
    }

    const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
--- a/src/model/cloud-runner/services/task-parameter-serializer.ts
+++ b/src/model/cloud-runner/services/task-parameter-serializer.ts
@@ -145,6 +145,7 @@ export class TaskParameterSerializer {
    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
+    array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');

    return array;
  }