Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)

* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
2026-02-03 23:49:08 +08:00 · 2023-01-20 17:40:57 +00:00
parent a45155c578
commit e334dc785a
39 changed files with 3083 additions and 399 deletions
--- a/src/model/cloud-runner/workflows/async-workflow.ts
+++ b/src/model/cloud-runner/workflows/async-workflow.ts
@@ -0,0 +1,60 @@
+import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
+import CloudRunnerLogger from '../services/cloud-runner-logger';
+import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import CloudRunner from '../cloud-runner';
+
+export class AsyncWorkflow {
+  public static async runAsyncWorkflow(
+    environmentVariables: CloudRunnerEnvironmentVariable[],
+    secrets: CloudRunnerSecret[],
+  ): Promise<string> {
+    try {
+      CloudRunnerLogger.log(`Cloud Runner is running async mode`);
+
+      let output = '';
+
+      output += await CloudRunner.Provider.runTaskInWorkflow(
+        CloudRunner.buildParameters.buildGuid,
+        `ubuntu`,
+        `apt-get update > /dev/null
+apt-get install -y curl tar tree npm git git-lfs jq git > /dev/null
+mkdir /builder
+printenv
+git config --global advice.detachedHead false
+git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"
+git config --global filter.lfs.process "git-lfs filter-process --skip"
+git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${CloudRunnerFolders.unityBuilderRepoUrl} /builder
+git clone -q -b ${CloudRunner.buildParameters.branch} ${CloudRunnerFolders.targetBuildRepoUrl} /repo
+cd /repo
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+./aws/install
+aws --version
+node /builder/dist/index.js -m async-workflow`,
+        `/${CloudRunnerFolders.buildVolumeFolder}`,
+        `/${CloudRunnerFolders.buildVolumeFolder}/`,
+        environmentVariables,
+        [
+          ...secrets,
+          ...[
+            {
+              ParameterKey: `AWS_ACCESS_KEY_ID`,
+              EnvironmentVariable: `AWS_ACCESS_KEY_ID`,
+              ParameterValue: process.env.AWS_ACCESS_KEY_ID || ``,
+            },
+            {
+              ParameterKey: `AWS_SECRET_ACCESS_KEY`,
+              EnvironmentVariable: `AWS_SECRET_ACCESS_KEY`,
+              ParameterValue: process.env.AWS_SECRET_ACCESS_KEY || ``,
+            },
+          ],
+        ],
+      );
+
+      return output;
+    } catch (error) {
+      throw error;
+    }
+  }
+}
--- a/src/model/cloud-runner/workflows/custom-workflow.ts
+++ b/src/model/cloud-runner/workflows/custom-workflow.ts
@@ -2,7 +2,8 @@ import CloudRunnerLogger from '../services/cloud-runner-logger';
 import CloudRunnerSecret from '../services/cloud-runner-secret';
 import { CloudRunnerFolders } from '../services/cloud-runner-folders';
 import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
-import { CloudRunnerCustomSteps, CustomStep } from '../services/cloud-runner-custom-steps';
+import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
+import { CustomStep } from '../services/custom-step';
 import CloudRunner from '../cloud-runner';

 export class CustomWorkflow {
@@ -26,9 +27,10 @@ export class CustomWorkflow {
    try {
      CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
      let output = '';
-      if (CloudRunner.buildParameters?.cloudRunnerDebug) {
-        CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
-      }
+
+      // if (CloudRunner.buildParameters?.cloudRunnerDebug) {
+      //   CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
+      // }
      for (const step of buildSteps) {
        output += await CloudRunner.Provider.runTaskInWorkflow(
          CloudRunner.buildParameters.buildGuid,
--- a/src/model/cloud-runner/workflows/workflow-composition-root.ts
+++ b/src/model/cloud-runner/workflows/workflow-composition-root.ts
@@ -3,10 +3,16 @@ import { CustomWorkflow } from './custom-workflow';
 import { WorkflowInterface } from './workflow-interface';
 import { BuildAutomationWorkflow } from './build-automation-workflow';
 import CloudRunner from '../cloud-runner';
+import CloudRunnerOptions from '../cloud-runner-options';
+import { AsyncWorkflow } from './async-workflow';

 export class WorkflowCompositionRoot implements WorkflowInterface {
  async run(cloudRunnerStepState: CloudRunnerStepState) {
    try {
+      if (CloudRunnerOptions.asyncCloudRunner) {
+        return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
+      }
+
      if (CloudRunner.buildParameters.customJob !== '') {
        return await CustomWorkflow.runCustomJobFromString(
          CloudRunner.buildParameters.customJob,