Cloud runner develop - latest fixes (#524)

2026-02-03 23:49:08 +08:00 · 2023-03-27 12:14:23 +01:00
parent 309d668d63
commit 7abb3a409d
91 changed files with 4943 additions and 3599 deletions
--- a/src/model/cloud-runner/workflows/async-workflow.ts
+++ b/src/model/cloud-runner/workflows/async-workflow.ts
@@ -1,7 +1,7 @@
-import CloudRunnerSecret from '../services/cloud-runner-secret';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
 import CloudRunner from '../cloud-runner';

 export class AsyncWorkflow {
@@ -11,6 +11,9 @@ export class AsyncWorkflow {
  ): Promise<string> {
    try {
      CloudRunnerLogger.log(`Cloud Runner is running async mode`);
+      const asyncEnvironmentVariable = new CloudRunnerEnvironmentVariable();
+      asyncEnvironmentVariable.name = `ASYNC_WORKFLOW`;
+      asyncEnvironmentVariable.value = `true`;

      let output = '';

@@ -34,7 +37,7 @@ aws --version
 node /builder/dist/index.js -m async-workflow`,
        `/${CloudRunnerFolders.buildVolumeFolder}`,
        `/${CloudRunnerFolders.buildVolumeFolder}/`,
-        environmentVariables,
+        [...environmentVariables, asyncEnvironmentVariable],
        [
          ...secrets,
          ...[
--- a/src/model/cloud-runner/workflows/build-automation-workflow.ts
+++ b/src/model/cloud-runner/workflows/build-automation-workflow.ts
@@ -1,26 +1,25 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
 import { WorkflowInterface } from './workflow-interface';
 import * as core from '@actions/core';
-import { CloudRunnerCustomHooks } from '../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../services/hooks/command-hook-service';
 import path from 'node:path';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerOptions from '../cloud-runner-options';
-import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
+import { ContainerHookService } from '../services/hooks/container-hook-service';

 export class BuildAutomationWorkflow implements WorkflowInterface {
-  async run(cloudRunnerStepState: CloudRunnerStepState) {
+  async run(cloudRunnerStepState: CloudRunnerStepParameters) {
    return await BuildAutomationWorkflow.standardBuildAutomation(cloudRunnerStepState.image, cloudRunnerStepState);
  }

-  private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepState) {
+  private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepParameters) {
    // TODO accept post and pre build steps as yaml files in the repo
    CloudRunnerLogger.log(`Cloud Runner is running standard build automation`);

    let output = '';

-    output += await CloudRunnerCustomSteps.RunPreBuildSteps(cloudRunnerStepState);
+    output += await ContainerHookService.RunPreBuildSteps(cloudRunnerStepState);
    CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');

    if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
@@ -40,7 +39,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
    if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
    CloudRunnerLogger.logWithTime('Build time');

-    output += await CloudRunnerCustomSteps.RunPostBuildSteps(cloudRunnerStepState);
+    output += await ContainerHookService.RunPostBuildSteps(cloudRunnerStepState);
    CloudRunnerLogger.logWithTime('Configurable post build step(s) time');

    CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
@@ -49,11 +48,11 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
  }

  private static get BuildWorkflow() {
-    const setupHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
-      x.step.includes(`setup`),
+    const setupHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
+      x.step?.includes(`setup`),
    );
-    const buildHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
-      x.step.includes(`build`),
+    const buildHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
+      x.step?.includes(`build`),
    );
    const builderPath = CloudRunnerFolders.ToLinuxFolder(
      path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`),
@@ -65,16 +64,14 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
      n 16.15.1 > /dev/null
      npm --version
      node --version
-      ${BuildAutomationWorkflow.TreeCommand}
      ${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
      export GITHUB_WORKSPACE="${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}"
+      df -H /data/
      ${BuildAutomationWorkflow.setupCommands(builderPath)}
      ${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
-      ${BuildAutomationWorkflow.TreeCommand}
      ${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
      ${BuildAutomationWorkflow.BuildCommands(builderPath)}
-      ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
-      ${BuildAutomationWorkflow.TreeCommand}`;
+      ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
  }

  private static setupCommands(builderPath: string) {
@@ -84,24 +81,17 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
      CloudRunnerFolders.unityBuilderRepoUrl
    } "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;

-    const retainedWorkspaceCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
-      CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
-    )}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
-      path.join(CloudRunnerFolders.repoPathAbsolute, `.git`),
-    )}" ]; then echo "Retained Workspace Already Exists!" ; fi`;
-
    const cloneBuilderCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
      CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
    )}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
      path.join(CloudRunnerFolders.builderPathAbsolute, `.git`),
-    )}" ]; then echo "Builder Already Exists!"; else ${commands}; fi`;
+    )}" ] ; then echo "Builder Already Exists!" && tree ${
+      CloudRunnerFolders.builderPathAbsolute
+    }; else ${commands} ; fi`;

    return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
-    echo "downloading game-ci..."
-    ${retainedWorkspaceCommands}
-    ${cloneBuilderCommands}
-    echo "bootstrap game ci cloud runner..."
-    node ${builderPath} -m remote-cli-pre-build`;
+${cloneBuilderCommands}
+node ${builderPath} -m remote-cli-pre-build`;
  }

  private static BuildCommands(builderPath: string) {
@@ -122,10 +112,4 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
    chmod +x ${builderPath}
    node ${builderPath} -m remote-cli-post-build`;
  }
-
-  private static get TreeCommand(): string {
-    return CloudRunnerOptions.cloudRunnerDebugTree
-      ? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s /${CloudRunnerFolders.buildVolumeFolder}/ && du -h -s ${CloudRunnerFolders.cacheFolderForAllFull}`
-      : `#`;
-  }
 }
--- a/src/model/cloud-runner/workflows/custom-workflow.ts
+++ b/src/model/cloud-runner/workflows/custom-workflow.ts
@@ -1,37 +1,37 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
-import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
-import { CustomStep } from '../services/custom-step';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
+import { ContainerHookService } from '../services/hooks/container-hook-service';
+import { ContainerHook } from '../services/hooks/container-hook';
 import CloudRunner from '../cloud-runner';

 export class CustomWorkflow {
-  public static async runCustomJobFromString(
+  public static async runContainerJobFromString(
    buildSteps: string,
    environmentVariables: CloudRunnerEnvironmentVariable[],
    secrets: CloudRunnerSecret[],
  ): Promise<string> {
-    return await CustomWorkflow.runCustomJob(
-      CloudRunnerCustomSteps.ParseSteps(buildSteps),
+    return await CustomWorkflow.runContainerJob(
+      ContainerHookService.ParseContainerHooks(buildSteps),
      environmentVariables,
      secrets,
    );
  }

-  public static async runCustomJob(
-    buildSteps: CustomStep[],
+  public static async runContainerJob(
+    steps: ContainerHook[],
    environmentVariables: CloudRunnerEnvironmentVariable[],
    secrets: CloudRunnerSecret[],
  ) {
    try {
-      CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
      let output = '';

      // if (CloudRunner.buildParameters?.cloudRunnerDebug) {
      //   CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
      // }
-      for (const step of buildSteps) {
+      for (const step of steps) {
+        CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
        output += await CloudRunner.Provider.runTaskInWorkflow(
          CloudRunner.buildParameters.buildGuid,
          step.image,
--- a/src/model/cloud-runner/workflows/workflow-composition-root.ts
+++ b/src/model/cloud-runner/workflows/workflow-composition-root.ts
@@ -1,20 +1,24 @@
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
 import { CustomWorkflow } from './custom-workflow';
 import { WorkflowInterface } from './workflow-interface';
 import { BuildAutomationWorkflow } from './build-automation-workflow';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 import { AsyncWorkflow } from './async-workflow';

 export class WorkflowCompositionRoot implements WorkflowInterface {
-  async run(cloudRunnerStepState: CloudRunnerStepState) {
+  async run(cloudRunnerStepState: CloudRunnerStepParameters) {
    try {
-      if (CloudRunnerOptions.asyncCloudRunner) {
+      if (
+        CloudRunnerOptions.asyncCloudRunner &&
+        !CloudRunner.isCloudRunnerAsyncEnvironment &&
+        !CloudRunner.isCloudRunnerEnvironment
+      ) {
        return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
      }

      if (CloudRunner.buildParameters.customJob !== '') {
-        return await CustomWorkflow.runCustomJobFromString(
+        return await CustomWorkflow.runContainerJobFromString(
          CloudRunner.buildParameters.customJob,
          cloudRunnerStepState.environment,
          cloudRunnerStepState.secrets,
@@ -22,7 +26,7 @@ export class WorkflowCompositionRoot implements WorkflowInterface {
      }

      return await new BuildAutomationWorkflow().run(
-        new CloudRunnerStepState(
+        new CloudRunnerStepParameters(
          cloudRunnerStepState.image.toString(),
          cloudRunnerStepState.environment,
          cloudRunnerStepState.secrets,
--- a/src/model/cloud-runner/workflows/workflow-interface.ts
+++ b/src/model/cloud-runner/workflows/workflow-interface.ts
@@ -1,8 +1,8 @@
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';

 export interface WorkflowInterface {
  run(
    // eslint-disable-next-line no-unused-vars
-    cloudRunnerStepState: CloudRunnerStepState,
+    cloudRunnerStepState: CloudRunnerStepParameters,
  ): Promise<string>;
 }