Cloud runner develop - latest fixes (#524)

Cloud runner develop - latest fixes (#524)

src/model/cloud-runner/options/cloud-runner-constants.ts

@@ -0,0 +1,4 @@
+class CloudRunnerConstants {
+  static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
+}
+export default CloudRunnerConstants;

src/model/cloud-runner/options/cloud-runner-environment-variable.ts

@@ -0,0 +1,5 @@
+class CloudRunnerEnvironmentVariable {
+  public name!: string;
+  public value!: string;
+}
+export default CloudRunnerEnvironmentVariable;

src/model/cloud-runner/options/cloud-runner-folders.ts

@@ -0,0 +1,90 @@
+import path from 'node:path';
+import CloudRunnerOptions from './cloud-runner-options';
+import CloudRunner from '../cloud-runner';
+import BuildParameters from '../../build-parameters';
+
+export class CloudRunnerFolders {
+  public static readonly repositoryFolder = 'repo';
+
+  public static ToLinuxFolder(folder: string) {
+    return folder.replace(/\\/g, `/`);
+  }
+
+  // Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
+
+  public static get uniqueCloudRunnerJobFolderAbsolute(): string {
+    return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
+      ? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
+      : path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
+  }
+
+  public static get cacheFolderForAllFull(): string {
+    return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
+  }
+
+  public static get cacheFolderForCacheKeyFull(): string {
+    return path.join(
+      '/',
+      CloudRunnerFolders.buildVolumeFolder,
+      CloudRunnerFolders.cacheFolder,
+      CloudRunner.buildParameters.cacheKey,
+    );
+  }
+
+  public static get builderPathAbsolute(): string {
+    return path.join(
+      CloudRunnerOptions.useSharedBuilder
+        ? `/${CloudRunnerFolders.buildVolumeFolder}`
+        : CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
+      `builder`,
+    );
+  }
+
+  public static get repoPathAbsolute(): string {
+    return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
+  }
+
+  public static get projectPathAbsolute(): string {
+    return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
+  }
+
+  public static get libraryFolderAbsolute(): string {
+    return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
+  }
+
+  public static get projectBuildFolderAbsolute(): string {
+    return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
+  }
+
+  public static get lfsFolderAbsolute(): string {
+    return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
+  }
+
+  public static get purgeRemoteCaching(): boolean {
+    return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
+  }
+
+  public static get lfsCacheFolderFull() {
+    return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
+  }
+
+  public static get libraryCacheFolderFull() {
+    return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
+  }
+
+  public static get unityBuilderRepoUrl(): string {
+    return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
+  }
+
+  public static get targetBuildRepoUrl(): string {
+    return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
+  }
+
+  public static get buildVolumeFolder() {
+    return 'data';
+  }
+
+  public static get cacheFolder() {
+    return 'cache';
+  }
+}

src/model/cloud-runner/options/cloud-runner-guid.ts

@@ -0,0 +1,11 @@
+import { customAlphabet } from 'nanoid';
+import CloudRunnerConstants from './cloud-runner-constants';
+
+class CloudRunnerNamespace {
+  static generateGuid(runNumber: string | number, platform: string) {
+    const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
+
+    return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
+  }
+}
+export default CloudRunnerNamespace;

src/model/cloud-runner/options/cloud-runner-options-reader.ts

@@ -0,0 +1,10 @@
+import Input from '../../input';
+import CloudRunnerOptions from './cloud-runner-options';
+
+class CloudRunnerOptionsReader {
+  static GetProperties() {
+    return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
+  }
+}
+
+export default CloudRunnerOptionsReader;

src/model/cloud-runner/options/cloud-runner-options.ts

@@ -0,0 +1,283 @@
+import { Cli } from '../../cli/cli';
+import CloudRunnerQueryOverride from './cloud-runner-query-override';
+import GitHub from '../../github';
+import * as core from '@actions/core';
+
+class CloudRunnerOptions {
+  // ### ### ###
+  // Input Handling
+  // ### ### ###
+  public static getInput(query: string): string | undefined {
+    if (GitHub.githubInputEnabled) {
+      const coreInput = core.getInput(query);
+      if (coreInput && coreInput !== '') {
+        return coreInput;
+      }
+    }
+    const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
+
+    // Query input sources
+    if (Cli.query(query, alternativeQuery)) {
+      return Cli.query(query, alternativeQuery);
+    }
+
+    if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
+      return CloudRunnerQueryOverride.query(query, alternativeQuery);
+    }
+
+    if (process.env[query] !== undefined) {
+      return process.env[query];
+    }
+
+    if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
+      return process.env[alternativeQuery];
+    }
+  }
+
+  public static ToEnvVarFormat(input: string): string {
+    if (input.toUpperCase() === input) {
+      return input;
+    }
+
+    return input
+      .replace(/([A-Z])/g, ' $1')
+      .trim()
+      .toUpperCase()
+      .replace(/ /g, '_');
+  }
+
+  // ### ### ###
+  // Provider parameters
+  // ### ### ###
+
+  static get region(): string {
+    return CloudRunnerOptions.getInput('region') || 'eu-west-2';
+  }
+
+  // ### ### ###
+  // GitHub  parameters
+  // ### ### ###
+  static get githubChecks(): boolean {
+    const value = CloudRunnerOptions.getInput('githubChecks');
+
+    return value === `true` || false;
+  }
+  static get githubCheckId(): string {
+    return CloudRunnerOptions.getInput('githubCheckId') || ``;
+  }
+
+  static get githubOwner(): string {
+    return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
+  }
+
+  static get githubRepoName(): string {
+    return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
+  }
+
+  static get finalHooks(): string[] {
+    return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
+  }
+
+  // ### ### ###
+  // Git syncronization parameters
+  // ### ### ###
+
+  static get githubRepo(): string | undefined {
+    return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
+  }
+  static get branch(): string {
+    if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
+      return (
+        CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
+      );
+    } else if (CloudRunnerOptions.getInput('branch')) {
+      return CloudRunnerOptions.getInput('branch') || ``;
+    } else {
+      return '';
+    }
+  }
+
+  // ### ### ###
+  // Cloud Runner parameters
+  // ### ### ###
+
+  static get buildPlatform(): string {
+    const input = CloudRunnerOptions.getInput('buildPlatform');
+    if (input) {
+      return input;
+    }
+    if (CloudRunnerOptions.providerStrategy !== 'local') {
+      return 'linux';
+    }
+
+    return ``;
+  }
+
+  static get cloudRunnerBranch(): string {
+    return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
+  }
+
+  static get providerStrategy(): string {
+    const provider =
+      CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
+    if (Cli.isCliMode) {
+      return provider || 'aws';
+    }
+
+    return provider || 'local';
+  }
+
+  static get containerCpu(): string {
+    return CloudRunnerOptions.getInput('containerCpu') || `1024`;
+  }
+
+  static get containerMemory(): string {
+    return CloudRunnerOptions.getInput('containerMemory') || `3072`;
+  }
+
+  static get customJob(): string {
+    return CloudRunnerOptions.getInput('customJob') || '';
+  }
+
+  // ### ### ###
+  // Custom commands from files parameters
+  // ### ### ###
+
+  static get containerHookFiles(): string[] {
+    return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
+  }
+
+  static get commandHookFiles(): string[] {
+    return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
+  }
+
+  // ### ### ###
+  // Custom commands from yaml parameters
+  // ### ### ###
+
+  static get commandHooks(): string {
+    return CloudRunnerOptions.getInput('commandHooks') || '';
+  }
+
+  static get postBuildContainerHooks(): string {
+    return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
+  }
+
+  static get preBuildContainerHooks(): string {
+    return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
+  }
+
+  // ### ### ###
+  // Input override handling
+  // ### ### ###
+
+  static get pullInputList(): string[] {
+    return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
+  }
+
+  static get inputPullCommand(): string {
+    const value = CloudRunnerOptions.getInput('inputPullCommand');
+
+    if (value === 'gcp-secret-manager') {
+      return 'gcloud secrets versions access 1 --secret="{0}"';
+    } else if (value === 'aws-secret-manager') {
+      return 'aws secretsmanager get-secret-value --secret-id {0}';
+    }
+
+    return value || '';
+  }
+
+  // ### ### ###
+  // Aws
+  // ### ### ###
+
+  static get awsStackName() {
+    return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
+  }
+
+  // ### ### ###
+  // K8s
+  // ### ### ###
+
+  static get kubeConfig(): string {
+    return CloudRunnerOptions.getInput('kubeConfig') || '';
+  }
+
+  static get kubeVolume(): string {
+    return CloudRunnerOptions.getInput('kubeVolume') || '';
+  }
+
+  static get kubeVolumeSize(): string {
+    return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
+  }
+
+  static get kubeStorageClass(): string {
+    return CloudRunnerOptions.getInput('kubeStorageClass') || '';
+  }
+
+  // ### ### ###
+  // Caching
+  // ### ### ###
+
+  static get cacheKey(): string {
+    return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
+  }
+
+  // ### ### ###
+  // Utility Parameters
+  // ### ### ###
+
+  static get cloudRunnerDebug(): boolean {
+    return (
+      CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
+      false
+    );
+  }
+  static get skipLfs(): boolean {
+    return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
+  }
+  static get skipCache(): boolean {
+    return CloudRunnerOptions.getInput(`skipCache`) === `true`;
+  }
+
+  public static get asyncCloudRunner(): boolean {
+    return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
+  }
+
+  public static get useLargePackages(): boolean {
+    return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
+  }
+
+  public static get useSharedBuilder(): boolean {
+    return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
+  }
+
+  public static get useCompressionStrategy(): boolean {
+    return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
+  }
+
+  public static get useCleanupCron(): boolean {
+    return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
+  }
+
+  // ### ### ###
+  // Retained Workspace
+  // ### ### ###
+
+  public static get maxRetainedWorkspaces(): string {
+    return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
+  }
+
+  // ### ### ###
+  // Garbage Collection
+  // ### ### ###
+
+  static get garbageMaxAge(): number {
+    return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
+  }
+}
+
+export default CloudRunnerOptions;

src/model/cloud-runner/options/cloud-runner-query-override.ts

@@ -0,0 +1,67 @@
+import Input from '../../input';
+import { GenericInputReader } from '../../input-readers/generic-input-reader';
+import CloudRunnerOptions from './cloud-runner-options';
+
+const formatFunction = (value: string, arguments_: any[]) => {
+  for (const element of arguments_) {
+    value = value.replace(`{${element.key}}`, element.value);
+  }
+
+  return value;
+};
+
+class CloudRunnerQueryOverride {
+  static queryOverrides: { [key: string]: string } | undefined;
+
+  // TODO accept premade secret sources or custom secret source definition yamls
+
+  public static query(key: string, alternativeKey: string) {
+    if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
+      return CloudRunnerQueryOverride.queryOverrides[key];
+    }
+    if (
+      CloudRunnerQueryOverride.queryOverrides &&
+      alternativeKey &&
+      CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
+    ) {
+      return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
+    }
+
+    return;
+  }
+
+  private static shouldUseOverride(query: string) {
+    if (CloudRunnerOptions.inputPullCommand !== '') {
+      if (CloudRunnerOptions.pullInputList.length > 0) {
+        const doesInclude =
+          CloudRunnerOptions.pullInputList.includes(query) ||
+          CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
+
+        return doesInclude ? true : false;
+      } else {
+        return true;
+      }
+    }
+  }
+
+  private static async queryOverride(query: string) {
+    if (!this.shouldUseOverride(query)) {
+      throw new Error(`Should not be trying to run override query on ${query}`);
+    }
+
+    return await GenericInputReader.Run(
+      formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
+    );
+  }
+
+  public static async PopulateQueryOverrideInput() {
+    const queries = CloudRunnerOptions.pullInputList;
+    CloudRunnerQueryOverride.queryOverrides = {};
+    for (const element of queries) {
+      if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
+        CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
+      }
+    }
+  }
+}
+export default CloudRunnerQueryOverride;

src/model/cloud-runner/options/cloud-runner-secret.ts

@@ -0,0 +1,6 @@
+class CloudRunnerSecret {
+  public ParameterKey!: string;
+  public EnvironmentVariable!: string;
+  public ParameterValue!: string;
+}
+export default CloudRunnerSecret;

src/model/cloud-runner/options/cloud-runner-statics.ts

@@ -0,0 +1,3 @@
+export class CloudRunnerStatics {
+  public static readonly logPrefix = `Cloud-Runner`;
+}

src/model/cloud-runner/options/cloud-runner-step-parameters.ts

@@ -0,0 +1,13 @@
+import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
+import CloudRunnerSecret from './cloud-runner-secret';
+
+export class CloudRunnerStepParameters {
+  public image: string;
+  public environment: CloudRunnerEnvironmentVariable[];
+  public secrets: CloudRunnerSecret[];
+  constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
+    this.image = image;
+    this.environment = environmentVariables;
+    this.secrets = secrets;
+  }
+}