Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)

* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
2026-02-04 08:09:08 +08:00 · 2023-01-20 17:40:57 +00:00
parent a45155c578
commit e334dc785a
39 changed files with 3083 additions and 399 deletions
--- a/src/model/cloud-runner/tests/cloud-runner-async-workflow.test.ts
+++ b/src/model/cloud-runner/tests/cloud-runner-async-workflow.test.ts
@@ -0,0 +1,33 @@
+import { BuildParameters, ImageTag } from '../..';
+import CloudRunner from '../cloud-runner';
+import UnityVersioning from '../../unity-versioning';
+import { Cli } from '../../cli/cli';
+import CloudRunnerOptions from '../cloud-runner-options';
+import setups from './cloud-runner-suite.test';
+
+async function CreateParameters(overrides) {
+  if (overrides) Cli.options = overrides;
+
+  return BuildParameters.create();
+}
+describe('Cloud Runner Async Workflows', () => {
+  setups();
+  it('Responds', () => {});
+
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
+    it('Async Workflows', async () => {
+      // Setup parameters
+      const buildParameter = await CreateParameters({
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.read('test-project'),
+        asyncCloudRunner: `true`,
+        githubChecks: `true`,
+      });
+      const baseImage = new ImageTag(buildParameter);
+
+      // Run the job
+      await CloudRunner.run(buildParameter, baseImage.toString());
+    }, 1_000_000_000);
+  }
+});
--- a/src/model/cloud-runner/tests/cloud-runner-run-once-custom-hooks.test.ts
+++ b/src/model/cloud-runner/tests/cloud-runner-run-once-custom-hooks.test.ts
@@ -24,6 +24,14 @@ describe('Cloud Runner Custom Hooks And Steps', () => {
 commands: echo "test"`;
    const yamlString2 = `- hook: before
  commands: echo "test"`;
+    const overrides = {
+      versioning: 'None',
+      projectPath: 'test-project',
+      unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+      targetPlatform: 'StandaloneLinux64',
+      cacheKey: `test-case-${uuidv4()}`,
+    };
+    CloudRunner.setup(await CreateParameters(overrides));
    const stringObject = CloudRunnerCustomSteps.ParseSteps(yamlString);
    const stringObject2 = CloudRunnerCustomSteps.ParseSteps(yamlString2);

--- a/src/model/cloud-runner/tests/cloud-runner-run-twice-caching.test.ts
+++ b/src/model/cloud-runner/tests/cloud-runner-run-twice-caching.test.ts
@@ -6,6 +6,7 @@ import CloudRunnerLogger from '../services/cloud-runner-logger';
 import { v4 as uuidv4 } from 'uuid';
 import CloudRunnerOptions from '../cloud-runner-options';
 import setups from './cloud-runner-suite.test';
+import * as fs from 'fs';

 async function CreateParameters(overrides) {
  if (overrides) {
@@ -45,6 +46,11 @@ describe('Cloud Runner Caching', () => {
      expect(results).not.toContain(cachePushFail);

      CloudRunnerLogger.log(`run 1 succeeded`);
+
+      if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
+        const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
+        expect(cacheFolderExists).toBeTruthy();
+      }
      const buildParameter2 = await CreateParameters(overrides);

      buildParameter2.cacheKey = buildParameter.cacheKey;
--- a/src/model/cloud-runner/tests/cloud-runner-run-twice-retaining.test.ts
+++ b/src/model/cloud-runner/tests/cloud-runner-run-twice-retaining.test.ts
@@ -6,7 +6,6 @@ import CloudRunnerLogger from '../services/cloud-runner-logger';
 import { v4 as uuidv4 } from 'uuid';
 import CloudRunnerOptions from '../cloud-runner-options';
 import setups from './cloud-runner-suite.test';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
 import * as fs from 'fs';
 import path from 'path';
 import { CloudRunnerFolders } from '../services/cloud-runner-folders';
@@ -46,6 +45,11 @@ describe('Cloud Runner Retain Workspace', () => {
      expect(results).toContain(buildSucceededString);
      expect(results).not.toContain(cachePushFail);

+      if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
+        const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
+        expect(cacheFolderExists).toBeTruthy();
+      }
+
      CloudRunnerLogger.log(`run 1 succeeded`);
      const buildParameter2 = await CreateParameters(overrides);

@@ -84,9 +88,6 @@ describe('Cloud Runner Retain Workspace', () => {
        CloudRunnerLogger.log(
          `Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
        );
-        await CloudRunnerSystem.Run(
-          `rm -r ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
-        );
      }
    });
  }
--- a/src/model/cloud-runner/tests/cloud-runner-s3-prebuilt-steps.test.ts
+++ b/src/model/cloud-runner/tests/cloud-runner-s3-prebuilt-steps.test.ts
@@ -38,7 +38,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
      expect(build2ContainsBuildSucceeded).toBeTruthy();

      const results = await CloudRunnerSystem.RunAndReadLines(
-        `aws s3 ls s3://game-ci-test-storage/cloud-runner-cache/${buildParameter2.cacheKey}/`,
+        `aws s3 ls s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/${buildParameter2.cacheKey}/`,
      );
      CloudRunnerLogger.log(results.join(`,`));
    }, 1_000_000_000);
--- a/src/model/cloud-runner/tests/shared-workspace-locking.test.ts
+++ b/src/model/cloud-runner/tests/shared-workspace-locking.test.ts
@@ -6,6 +6,7 @@ import { v4 as uuidv4 } from 'uuid';
 import CloudRunnerOptions from '../cloud-runner-options';
 import UnityVersioning from '../../unity-versioning';
 import BuildParameters from '../../build-parameters';
+import CloudRunner from '../cloud-runner';

 async function CreateParameters(overrides) {
  if (overrides) {
@@ -32,6 +33,7 @@ describe('Cloud Runner Locking', () => {

      const newWorkspaceName = `test-workspace-${uuidv4()}`;
      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
      await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
      const isExpectedUnlockedBeforeLocking =
        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;