Cloud runner develop - Stabilizes kubernetes provider (#531)

* fixes * fixes * fixes * fixes * fixes * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * Update cloud-runner-ci-pipeline.yml * Update cloud-runner-ci-pipeline.yml * no storage class specified * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * updates * log file path * latest develop * log file path * log file path * Update package.json * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * update pipeline to use k3s * version: 'latest' * fixes * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * no wait, just repeat logs * no wait, just repeat logs * remove typo - double await * test fix - kubernetes - name typo in github yaml * test fix - kubernetes - name typo in github yaml * check missing log file * check missing log file * Push to steam test * Push to steam test * Fix path * k8s reliable log hashing * k8s reliable log hashing * k8s reliable log hashing * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Fix exit flow for k8s job * hash comparison logging for log complete in k8s flow * Interrupt k8s logs when logs found * cleanup async parameter * cleanup async parameter * cleanup async parameter * fixes * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix
2026-02-05 17:29:09 +08:00 · 2024-02-06 23:46:31 +00:00
parent 2800d14403
commit e73b48fb38
47 changed files with 1456 additions and 856 deletions
--- a/src/model/cloud-runner/remote-client/caching.ts
+++ b/src/model/cloud-runner/remote-client/caching.ts
@@ -80,7 +80,7 @@ export class Caching {
      }

      await CloudRunnerSystem.Run(
-        `tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path.basename(sourceFolder)}`,
+        `tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
      );
      await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
      assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
--- a/src/model/cloud-runner/remote-client/index.ts
+++ b/src/model/cloud-runner/remote-client/index.ts
@@ -12,16 +12,83 @@ import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import YAML from 'yaml';
 import GitHub from '../../github';
 import BuildParameters from '../../build-parameters';
+import { Cli } from '../../cli/cli';
+import CloudRunnerOptions from '../options/cloud-runner-options';

 export class RemoteClient {
  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-  static async runRemoteClientJob() {
+  static async setupRemoteClient() {
    CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
    if (!(await RemoteClient.handleRetainedWorkspace())) {
      await RemoteClient.bootstrapRepository();
    }
+    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
    await RemoteClient.runCustomHookFiles(`before-build`);
  }
+
+  @CliFunction('remote-cli-log-stream', `log stream from standard input`)
+  public static async remoteClientLogStream() {
+    const logFile = Cli.options!['logFile'];
+    process.stdin.resume();
+    process.stdin.setEncoding('utf8');
+
+    let lingeringLine = '';
+
+    process.stdin.on('data', (chunk) => {
+      const lines = chunk.toString().split('\n');
+
+      lines[0] = lingeringLine + lines[0];
+      lingeringLine = lines.pop() || '';
+
+      for (const element of lines) {
+        if (CloudRunnerOptions.providerStrategy !== 'k8s') {
+          CloudRunnerLogger.log(element);
+        } else {
+          fs.appendFileSync(logFile, element);
+          CloudRunnerLogger.log(element);
+        }
+      }
+    });
+
+    process.stdin.on('end', () => {
+      if (CloudRunnerOptions.providerStrategy !== 'k8s') {
+        CloudRunnerLogger.log(lingeringLine);
+      } else {
+        fs.appendFileSync(logFile, lingeringLine);
+        CloudRunnerLogger.log(lingeringLine);
+      }
+    });
+  }
+
+  @CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
+  public static async remoteClientPostBuild(): Promise<string> {
+    RemoteClientLogger.log(`Running POST build tasks`);
+
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
+      `lib-${CloudRunner.buildParameters.buildGuid}`,
+    );
+
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
+      `build-${CloudRunner.buildParameters.buildGuid}`,
+    );
+
+    if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
+      await CloudRunnerSystem.Run(
+        `rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
+      );
+    }
+
+    await RemoteClient.runCustomHookFiles(`after-build`);
+
+    // WIP - need to give the pod permissions to create config map
+    await RemoteClientLogger.handleLogManagementPostJob();
+
+    return new Promise((result) => result(``));
+  }
  static async runCustomHookFiles(hookLifecycle: string) {
    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
@@ -47,7 +114,6 @@ export class RemoteClient {
      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
    );
    await RemoteClient.cloneRepoWithoutLFSFiles();
-    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
    await RemoteClient.sizeOfFolder(
      'repo before lfs cache pull',
      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
--- a/src/model/cloud-runner/remote-client/remote-client-logger.ts
+++ b/src/model/cloud-runner/remote-client/remote-client-logger.ts
@@ -1,8 +1,18 @@
 import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import fs from 'node:fs';
+import path from 'node:path';
+import CloudRunner from '../cloud-runner';
+import CloudRunnerOptions from '../options/cloud-runner-options';

 export class RemoteClientLogger {
+  private static get LogFilePath() {
+    return path.join(`/home`, `job-log.txt`);
+  }
+
  public static log(message: string) {
-    CloudRunnerLogger.log(`[Client] ${message}`);
+    const finalMessage = `[Client] ${message}`;
+    this.appendToFile(finalMessage);
+    CloudRunnerLogger.log(finalMessage);
  }

  public static logCliError(message: string) {
@@ -16,4 +26,57 @@ export class RemoteClientLogger {
  public static logWarning(message: string) {
    CloudRunnerLogger.logWarning(message);
  }
+
+  public static appendToFile(message: string) {
+    if (CloudRunner.isCloudRunnerEnvironment) {
+      fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
+    }
+  }
+
+  public static async handleLogManagementPostJob() {
+    if (CloudRunnerOptions.providerStrategy !== 'k8s') {
+      return;
+    }
+    CloudRunnerLogger.log(`Collected Logs`);
+
+    // check for log file not existing
+    if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
+      CloudRunnerLogger.log(`Log file does not exist`);
+
+      // check if CloudRunner.isCloudRunnerEnvironment is true, log
+      if (!CloudRunner.isCloudRunnerEnvironment) {
+        CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
+      }
+
+      return;
+    }
+    CloudRunnerLogger.log(`Log file exist`);
+    await new Promise((resolve) => setTimeout(resolve, 1));
+
+    // let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
+    //
+    // hashedLogs = md5(hashedLogs);
+    //
+    // for (let index = 0; index < 3; index++) {
+    //   CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
+    //   const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
+    //   CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
+    //   CloudRunnerLogger.log(
+    //     `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
+    //   );
+    //
+    //   // wait for 15 seconds to allow the log to be sent
+    //   await new Promise((resolve) => setTimeout(resolve, 15000));
+    // }
+  }
+  public static HandleLog(message: string): boolean {
+    if (RemoteClientLogger.value !== '') {
+      RemoteClientLogger.value += `\n`;
+    }
+
+    RemoteClientLogger.value += message;
+
+    return false;
+  }
+  static value: string = '';
 }