Fix Android cmdline-tools path

* `manualExit` suppresses `-quit`, useful for buildMethods with async calls

* Use boolean

.eslintrc.json

@@ -1,11 +1,22 @@
 {
-  "plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
+  "plugins": [
-  "extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
+    "jest",
+    "@typescript-eslint",
+    "prettier",
+    "unicorn"
+  ],
+  "extends": [
+    "plugin:unicorn/recommended",
+    "plugin:github/recommended",
+    "plugin:prettier/recommended"
+  ],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
     "ecmaVersion": 2020,
     "sourceType": "module",
-    "extraFileExtensions": [".mjs"],
+    "extraFileExtensions": [
+      ".mjs"
+    ],
     "ecmaFeatures": {
       "impliedStrict": true
     },
@@ -22,7 +33,10 @@
     // Namespaces or sometimes needed
     "import/no-namespace": "off",
     // Properly format comments
-    "spaced-comment": ["error", "always"],
+    "spaced-comment": [
+      "error",
+      "always"
+    ],
     "lines-around-comment": [
       "error",
       {
@@ -38,14 +52,31 @@
     // Mandatory spacing
     "padding-line-between-statements": [
       "error",
-      { "blankLine": "always", "prev": "*", "next": "return" },
+      {
-      { "blankLine": "always", "prev": "directive", "next": "*" },
+        "blankLine": "always",
-      { "blankLine": "any", "prev": "directive", "next": "directive" }
+        "prev": "*",
+        "next": "return"
+      },
+      {
+        "blankLine": "always",
+        "prev": "directive",
+        "next": "*"
+      },
+      {
+        "blankLine": "any",
+        "prev": "directive",
+        "next": "directive"
+      }
     ],
     // Enforce camelCase
     "camelcase": "error",
     // Allow forOfStatements
-    "no-restricted-syntax": ["error", "ForInStatement", "LabeledStatement", "WithStatement"],
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement"
+    ],
     // Continue is viable in forOf loops in generators
     "no-continue": "off",
     // From experience, named exports are almost always desired. I got tired of this rule
@@ -53,8 +84,17 @@
     // Unused vars are useful to keep method signatures consistent and documented
     "@typescript-eslint/no-unused-vars": "off",
     // For this project only use kebab-case
-    "unicorn/filename-case": ["error", { "cases": { "kebabCase": true } }],
+    "unicorn/filename-case": [
+      "error",
+      {
+        "cases": {
+          "kebabCase": true
+        }
+      }
+    ],
     // Allow Array.from(set) mitigate TS2569 which would require '--downlevelIteration'
-    "unicorn/prefer-spread": "off"
+    "unicorn/prefer-spread": "off",
+    // Temp disable to prevent mixing changes with other PRs
+    "i18n-text/no-en": "off"
   }
 }

.github/pull_request_template.md

@@ -8,5 +8,7 @@
 
 - [x] Read the contribution [guide](https://github.com/game-ci/unity-builder/blob/main/CONTRIBUTING.md) and accept the
       [code](https://github.com/game-ci/unity-builder/blob/main/CODE_OF_CONDUCT.md) of conduct
+- [ ] Docs (If new inputs or outputs have been added or changes to behavior that should be documented. Please make
+            a PR in the [documentation repo](https://github.com/game-ci/documentation))
 - [ ] Readme (updated or not needed)
 - [ ] Tests (added, updated or not needed)

.github/workflows/build-tests-mac.yml

@@ -0,0 +1,80 @@
+name: Builds - MacOS
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  buildForAllPlatformsWindows:
+    name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        projectPath:
+          - test-project
+        unityVersion:
+          - 2021.3.29f1
+          - 2022.1.24f1
+          - 2022.2.21f1
+          - 2022.3.7f1
+          - 2023.1.8f1
+        targetPlatform:
+          - StandaloneOSX # Build a MacOS executable
+
+    steps:
+      ###########################
+      #         Checkout        #
+      ###########################
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+
+      ###########################
+      #          Cache          #
+      ###########################
+      - uses: actions/cache@v3
+        with:
+          path: ${{ matrix.projectPath }}/Library
+          key: Library-${{ matrix.projectPath }}-macos-${{ matrix.targetPlatform }}
+          restore-keys: |
+            Library-${{ matrix.projectPath }}-macos-
+            Library-
+
+      ###########################
+      #  Set Scripting Backend  #
+      ###########################
+      - name: Set Scripting Backend To il2cpp
+        run: |
+          mv -f "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" "./test-project/ProjectSettings/ProjectSettings.asset"
+
+      ###########################
+      #          Build          #
+      ###########################
+      - uses: ./
+        env:
+          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
+          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
+          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
+        with:
+          projectPath: ${{ matrix.projectPath }}
+          unityVersion: ${{ matrix.unityVersion }}
+          targetPlatform: ${{ matrix.targetPlatform }}
+          customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
+          # We use dirty build because we are replacing the default project settings file above
+          allowDirtyBuild: true
+
+      ###########################
+      #          Upload         #
+      ###########################
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Build MacOS (${{ matrix.unityVersion }})
+          path: build
+          retention-days: 14

.github/workflows/build-tests-ubuntu.yml

@@ -1,12 +1,18 @@
-name: Builds
+name: Builds - Ubuntu
 
 on:
   workflow_dispatch:
-  push: { branches: [main] }
+  push:
+    branches:
+      - main
   pull_request:
     paths-ignore:
       - '.github/**'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   UNITY_LICENSE:
     "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root>\n    <License
@@ -35,31 +41,22 @@ env:
 
 jobs:
   buildForAllPlatformsUbuntu:
-    name: Build for ${{ matrix.targetPlatform }} on version ${{ matrix.unityVersion }}
+    name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        providerStrategy:
           # - local-docker
           - local
         projectPath:
           - test-project
         unityVersion:
-          - 2018.3.14f1
+          - 2021.3.29f1
-          - 2018.4.36f1
-          - 2019.1.14f1
-          - 2019.2.21f1
-          - 2019.3.15f1
-          - 2019.4.40f1
-          - 2020.1.17f1
-          - 2020.2.7f1
-          - 2020.3.44f1
-          - 2021.1.28f1
-          - 2021.2.19f1
-          - 2021.3.18f1
           - 2022.1.24f1
-          - 2022.2.6f1
+          - 2022.2.21f1
+          - 2022.3.7f1
+          - 2023.1.8f1
         targetPlatform:
           - StandaloneOSX # Build a macOS standalone (Intel 64-bit) with mono backend.
           - StandaloneWindows64 # Build a Windows 64-bit standalone with mono backend.
@@ -101,7 +98,7 @@ jobs:
           unityVersion: ${{ matrix.unityVersion }}
           targetPlatform: ${{ matrix.targetPlatform }}
           customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
 
       ###########################
       #          Upload         #

.github/workflows/build-tests-windows.yml

@@ -1,4 +1,4 @@
-name: Windows Builds
+name: Builds - Windows
 
 on:
   workflow_dispatch:
@@ -6,35 +6,13 @@ on:
     branches:
       - main
 
-env:
+concurrency:
-  UNITY_LICENSE:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-    "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root>\n    <License
+  cancel-in-progress: true
-    id=\"Terms\">\n        <MachineBindings>\n            <Binding Key=\"1\"
-    Value=\"576562626572264761624c65526f7578\"/>\n            <Binding Key=\"2\"
-    Value=\"576562626572264761624c65526f7578\"/>\n        </MachineBindings>\n        <MachineID
-    Value=\"D7nTUnjNAmtsUMcnoyrqkgIbYdM=\"/>\n        <SerialHash
-    Value=\"2033b8ac3e6faa3742ca9f0bfae44d18f2a96b80\"/>\n        <Features>\n            <Feature
-    Value=\"33\"/>\n            <Feature Value=\"1\"/>\n            <Feature Value=\"12\"/>\n            <Feature
-    Value=\"2\"/>\n            <Feature Value=\"24\"/>\n            <Feature Value=\"3\"/>\n            <Feature
-    Value=\"36\"/>\n            <Feature Value=\"17\"/>\n            <Feature Value=\"19\"/>\n            <Feature
-    Value=\"62\"/>\n        </Features>\n        <DeveloperData
-    Value=\"AQAAAEY0LUJHUlgtWEQ0RS1aQ1dWLUM1SlctR0RIQg==\"/>\n        <SerialMasked
-    Value=\"F4-BGRX-XD4E-ZCWV-C5JW-XXXX\"/>\n        <StartDate Value=\"2021-02-08T00:00:00\"/>\n        <UpdateDate
-    Value=\"2021-02-09T00:34:57\"/>\n        <InitialActivationDate
-    Value=\"2021-02-08T00:34:56\"/>\n        <LicenseVersion Value=\"6.x\"/>\n        <ClientProvidedVersion
-    Value=\"2018.4.30f1\"/>\n        <AlwaysOnline Value=\"false\"/>\n        <Entitlements>\n            <Entitlement
-    Ns=\"unity_editor\" Tag=\"UnityPersonal\" Type=\"EDITOR\"
-    ValidTo=\"9999-12-31T00:00:00\"/>\n            <Entitlement Ns=\"unity_editor\" Tag=\"DarkSkin\"
-    Type=\"EDITOR_FEATURE\" ValidTo=\"9999-12-31T00:00:00\"/>\n        </Entitlements>\n    </License>\n<Signature
-    xmlns=\"http://www.w3.org/2000/09/xmldsig#\"><SignedInfo><CanonicalizationMethod
-    Algorithm=\"http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments\"/><SignatureMethod
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#rsa-sha1\"/><Reference URI=\"#Terms\"><Transforms><Transform
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#enveloped-signature\"/></Transforms><DigestMethod
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#sha1\"/><DigestValue>m0Db8UK+ktnOLJBtHybkfetpcKo=</DigestValue></Reference></SignedInfo><SignatureValue>o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==</SignatureValue></Signature></root>"
 
 jobs:
   buildForAllPlatformsWindows:
-    name: Build for ${{ matrix.targetPlatform }} on version ${{ matrix.unityVersion }}
+    name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
     runs-on: windows-2019
     strategy:
       fail-fast: false
@@ -42,14 +20,11 @@ jobs:
         projectPath:
           - test-project
         unityVersion:
-          - 2019.3.15f1 # Minimum version for IL2CPP
+          - 2021.3.29f1
-          - 2019.4.40f1
-          - 2020.1.17f1
-          - 2020.2.7f1
-          - 2020.3.44f1
-          - 2021.3.18f1 # 2021.1 and 2021.2 seem to have IL2CPP issues
           - 2022.1.24f1
-          - 2022.2.6f1
+          - 2022.2.21f1
+          - 2022.3.7f1
+          - 2023.1.8f1
         targetPlatform:
           - StandaloneWindows64 # Build a Windows 64-bit standalone.
           - StandaloneWindows # Build a Windows 32-bit standalone.
@@ -89,7 +64,7 @@ jobs:
         uses: ./
         id: build-1
         continue-on-error: true
-        timeout-minutes: 60
+        timeout-minutes: 30
         env:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
@@ -111,7 +86,7 @@ jobs:
         uses: ./
         id: build-2
         continue-on-error: true
-        timeout-minutes: 60
+        timeout-minutes: 30
         if: steps.build-1.outcome == 'failure'
         env:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
@@ -133,7 +108,7 @@ jobs:
       - name: Build Retry 2
         uses: ./
         id: build-3
-        timeout-minutes: 60
+        timeout-minutes: 30
         if: ${{ steps.build-1.outcome == 'failure' && steps.build-2.outcome == 'failure' }}
         env:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}

.github/workflows/cleanup.yml

@@ -21,7 +21,7 @@ jobs:
           lfs: true
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: '18'
       - run: yarn
       - run: yarn run cli --help
         env:

.github/workflows/cloud-runner-async-checks.yml

@@ -23,7 +23,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-github-pipelines
+  AWS_STACK_NAME: game-ci-github-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
   CLOUD_RUNNER_DEBUG: true
   CLOUD_RUNNER_DEBUG_TREE: true
@@ -39,20 +39,21 @@ jobs:
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout (default)
+      - timeout-minutes: 180
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-      - run: yarn
-      - run: yarn run cli -m checks-update
-        timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
           CLOUD_RUNNER_CLUSTER: local-docker
-          AWS_BASE_STACK_NAME: game-ci-github-pipelines
+          AWS_STACK_NAME: game-ci-github-pipelines
           CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
+        run: |
+          git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
+          cd unity-builder
+          yarn
+          ls
+          yarn run cli -m checks-update

.github/workflows/cloud-runner-ci-pipeline.yml

@@ -21,81 +21,140 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-team-pipelines
+  AWS_STACK_NAME: game-ci-team-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-  CLOUD_RUNNER_DEBUG: true
-  CLOUD_RUNNER_DEBUG_TREE: true
   DEBUG: true
   UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
   PROJECT_PATH: test-project
   UNITY_VERSION: 2019.3.15f1
   USE_IL2CPP: false
   USE_GKE_GCLOUD_AUTH_PLUGIN: true
+  GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  integrationTests:
+  smokeTests:
-    name: Integration Tests
+    name: Smoke Tests
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        test:
-          - aws
+          #- 'cloud-runner-async-workflow'
+          - 'cloud-runner-caching'
+          # - 'cloud-runner-end2end-caching'
+          # - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          - 'cloud-runner-hooks'
+          - 'cloud-runner-local-persistence'
+          - 'cloud-runner-locking-core'
+          - 'cloud-runner-locking-get-locked'
+        providerStrategy:
+          #- aws
           - local-docker
-          - k8s
+          #- k8s
     steps:
       - name: Checkout (default)
         uses: actions/checkout@v3
         with:
           lfs: false
-      - uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v1'
-      - name: Get GKE cluster credentials
-        run: |
-          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
-          gcloud components install gke-gcloud-auth-plugin
-          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
       - run: yarn
-      - run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster != 'local-docker'
+        timeout-minutes: 35
-        timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.providerStrategy }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  tests:
-      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
+    # needs:
-        if: matrix.CloudRunnerCluster == 'local-docker'
+    #   - smokeTests
-        timeout-minutes: 180
+    #   - buildTargetTests
+    name: Integration Tests
+    if: github.event.event_type != 'pull_request_target'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        providerStrategy:
+          - aws
+          - local-docker
+          - k8s
+        test:
+          - 'cloud-runner-async-workflow'
+          #- 'cloud-runner-caching'
+          - 'cloud-runner-end2end-locking'
+          - 'cloud-runner-end2end-caching'
+          - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          #- 'cloud-runner-hooks'
+          - 'cloud-runner-s3-steps'
+          #- 'cloud-runner-local-persistence'
+          #- 'cloud-runner-locking-core'
+          #- 'cloud-runner-locking-get-locked'
+    steps:
+      - name: Checkout (default)
+        uses: actions/checkout@v2
+        with:
+          lfs: false
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+      - run: yarn
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 60
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          PROVIDER_STRATEGY: ${{ matrix.providerStrategy }}
-  localBuildTests:
+  buildTargetTests:
     name: Local Build Target Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        providerStrategy:
           #- aws
           - local-docker
           #- k8s
@@ -114,20 +173,18 @@ jobs:
       - run: yarn
       - uses: ./
         id: unity-build
-        timeout-minutes: 90
+        timeout-minutes: 30
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
         with:
           cloudRunnerTests: true
           versioning: None
-          projectPath: test-project
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
           targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
       - run: |
           cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
           path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
           retention-days: 14

.github/workflows/integrity-check.yml

@@ -7,6 +7,10 @@ on:
 env:
   CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     name: Tests
@@ -15,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '18'
       - run: yarn
       - run: yarn lint
       - run: yarn test --coverage

.github/workflows/mac-build-tests.yml

@@ -1,106 +0,0 @@
-name: Mac Builds
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-
-env:
-  UNITY_LICENSE:
-    "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root>\n    <License
-    id=\"Terms\">\n        <MachineBindings>\n            <Binding Key=\"1\"
-    Value=\"576562626572264761624c65526f7578\"/>\n            <Binding Key=\"2\"
-    Value=\"576562626572264761624c65526f7578\"/>\n        </MachineBindings>\n        <MachineID
-    Value=\"D7nTUnjNAmtsUMcnoyrqkgIbYdM=\"/>\n        <SerialHash
-    Value=\"2033b8ac3e6faa3742ca9f0bfae44d18f2a96b80\"/>\n        <Features>\n            <Feature
-    Value=\"33\"/>\n            <Feature Value=\"1\"/>\n            <Feature Value=\"12\"/>\n            <Feature
-    Value=\"2\"/>\n            <Feature Value=\"24\"/>\n            <Feature Value=\"3\"/>\n            <Feature
-    Value=\"36\"/>\n            <Feature Value=\"17\"/>\n            <Feature Value=\"19\"/>\n            <Feature
-    Value=\"62\"/>\n        </Features>\n        <DeveloperData
-    Value=\"AQAAAEY0LUJHUlgtWEQ0RS1aQ1dWLUM1SlctR0RIQg==\"/>\n        <SerialMasked
-    Value=\"F4-BGRX-XD4E-ZCWV-C5JW-XXXX\"/>\n        <StartDate Value=\"2021-02-08T00:00:00\"/>\n        <UpdateDate
-    Value=\"2021-02-09T00:34:57\"/>\n        <InitialActivationDate
-    Value=\"2021-02-08T00:34:56\"/>\n        <LicenseVersion Value=\"6.x\"/>\n        <ClientProvidedVersion
-    Value=\"2018.4.30f1\"/>\n        <AlwaysOnline Value=\"false\"/>\n        <Entitlements>\n            <Entitlement
-    Ns=\"unity_editor\" Tag=\"UnityPersonal\" Type=\"EDITOR\"
-    ValidTo=\"9999-12-31T00:00:00\"/>\n            <Entitlement Ns=\"unity_editor\" Tag=\"DarkSkin\"
-    Type=\"EDITOR_FEATURE\" ValidTo=\"9999-12-31T00:00:00\"/>\n        </Entitlements>\n    </License>\n<Signature
-    xmlns=\"http://www.w3.org/2000/09/xmldsig#\"><SignedInfo><CanonicalizationMethod
-    Algorithm=\"http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments\"/><SignatureMethod
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#rsa-sha1\"/><Reference URI=\"#Terms\"><Transforms><Transform
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#enveloped-signature\"/></Transforms><DigestMethod
-    Algorithm=\"http://www.w3.org/2000/09/xmldsig#sha1\"/><DigestValue>m0Db8UK+ktnOLJBtHybkfetpcKo=</DigestValue></Reference></SignedInfo><SignatureValue>o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==</SignatureValue></Signature></root>"
-
-jobs:
-  buildForAllPlatformsWindows:
-    name: Build for ${{ matrix.targetPlatform }} on version ${{ matrix.unityVersion }}
-    runs-on: macos-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        projectPath:
-          - test-project
-        unityVersion:
-          - 2019.4.40f1 # Minimum version for IL2CPP
-          - 2020.1.17f1
-          - 2020.2.7f1
-          - 2020.3.44f1
-          - 2021.1.28f1
-          - 2021.2.19f1
-          - 2021.3.18f1
-          - 2022.1.24f1
-          - 2022.2.6f1
-        targetPlatform:
-          - StandaloneOSX # Build a MacOS executable
-
-    steps:
-      ###########################
-      #         Checkout        #
-      ###########################
-      - uses: actions/checkout@v3
-        with:
-          lfs: true
-
-      ###########################
-      #          Cache          #
-      ###########################
-      - uses: actions/cache@v3
-        with:
-          path: ${{ matrix.projectPath }}/Library
-          key: Library-${{ matrix.projectPath }}-macos-${{ matrix.targetPlatform }}
-          restore-keys: |
-            Library-${{ matrix.projectPath }}-macos-
-            Library-
-
-      ###########################
-      #  Set Scripting Backend  #
-      ###########################
-      - name: Set Scripting Backend To il2cpp
-        run: |
-          mv -f "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" "./test-project/ProjectSettings/ProjectSettings.asset"
-
-      ###########################
-      #          Build          #
-      ###########################
-      - uses: ./
-        env:
-          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
-          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
-          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
-        with:
-          projectPath: ${{ matrix.projectPath }}
-          unityVersion: ${{ matrix.unityVersion }}
-          targetPlatform: ${{ matrix.targetPlatform }}
-          customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
-          # We use dirty build because we are replacing the default project settings file above
-          allowDirtyBuild: true
-
-      ###########################
-      #          Upload         #
-      ###########################
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Build MacOS (${{ matrix.unityVersion }})
-          path: build
-          retention-days: 14

.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Jest Test",
+      "program": "${workspaceRoot}/node_modules/jest/bin/jest.js",
+      "args": [
+        "--collectCoverage=false",
+        "--colors",
+        "--config",
+        "${workspaceRoot}/jest.config.js",
+        "--runInBand",
+        "--runTestsByPath",
+        "${relativeFile}",
+        "--testPathPattern=${fileDirname}",
+        "--testTimeout=10000000"
+      ],
+      "outputCapture": "std",
+      "internalConsoleOptions": "openOnSessionStart",
+      "envFile": "${workspaceRoot}/.env",
+      "skipFiles": ["${workspaceRoot}/../../node_modules/**/*", "<node_internals>/**/*"]
+    }
+  ]
+}

README.md

@@ -10,8 +10,9 @@ Part of the <a href="https://game.ci">GameCI</a> open source project.
 <br />
 <br />
 
-[![Actions status](https://github.com/game-ci/unity-builder/workflows/Builds/badge.svg?event=push&branch=main)](https://github.com/game-ci/unity-builder/actions?query=branch%3Amain+event%3Apush+workflow%3A%22Builds)
+[![Builds - Ubuntu](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-ubuntu.yml/badge.svg)](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-ubuntu.yml)
-[![lgtm - code quality](https://img.shields.io/lgtm/grade/javascript/g/webbertakken/unity-builder.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/webbertakken/unity-builder/context:javascript)
+[![Builds - Windows](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-windows.yml/badge.svg)](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-windows.yml)
+[![Builds - MacOS](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-mac.yml/badge.svg)](https://github.com/game-ci/unity-builder/actions/workflows/build-tests-mac.yml)
 [![codecov - test coverage](https://codecov.io/gh/game-ci/unity-builder/branch/master/graph/badge.svg)](https://codecov.io/gh/game-ci/unity-builder)
 <br />
 <br />

action.yml

@@ -31,6 +31,10 @@ inputs:
     required: false
     default: ''
     description: 'Path to a Namespace.Class.StaticMethod to run to perform the build.'
+  manualExit:
+    required: false
+    default: ''
+    description: 'Suppresses `-quit`. Exit your build method using `EditorApplication.Exit(0)` instead.'
   customParameters:
     required: false
     default: ''
@@ -47,12 +51,9 @@ inputs:
     required: false
     default: ''
     description: 'The android versionCode'
-  androidAppBundle:
-    required: false
-    default: 'false'
-    description: '[Deprecated] Use androidExportType instead. Whether to build .aab instead of .apk'
   androidExportType:
     required: false
+    default: 'androidPackage'
     description:
       'The android export type. Should be androidPackage for apk, androidAppBundle for aab, or androidStudioProject for
       an android studio project.'
@@ -86,8 +87,12 @@ inputs:
     description: 'The android symbol type to export. Should be "none", "public" or "debugging".'
   sshAgent:
     required: false
-    default: 'public'
+    default: ''
     description: 'SSH Agent path to forward to the container'
+  sshPublicKeysDirectoryPath:
+    required: false
+    default: ''
+    description: 'Path to a directory containing SSH public keys to forward to the container.'
   gitPrivateToken:
     required: false
     default: ''
@@ -117,7 +122,7 @@ inputs:
     description:
       '[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  customStepFiles:
+  containerHookFiles:
     required: false
     default: ''
     description:
@@ -129,7 +134,7 @@ inputs:
     description:
       '[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
       match a yaml step file inside your repo in the folder .game-ci/hooks/'
-  customJobHooks:
+  customCommandHooks:
     required: false
     default: ''
     description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
@@ -139,11 +144,11 @@ inputs:
     description:
       '[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  awsBaseStackName:
+  awsStackName:
     default: 'game-ci'
     required: false
     description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
-  cloudRunnerCluster:
+  providerStrategy:
     default: 'local'
     required: false
     description:
@@ -206,6 +211,16 @@ inputs:
     description:
       'The version of Unity Hub to install on MacOS (e.g. 3.4.0). Defaults to latest available on brew if empty string
       or nothing is specified.'
+  unityLicensingServer:
+    default: ''
+    required: false
+    description: 'The Unity licensing server address to use for activating Unity.'
+  dockerWorkspacePath:
+    default: '/github/workspace'
+    required: false
+    description:
+      'The path to mount the workspace inside the docker container. For windows, leave out the drive letter. For example
+      c:/github/workspace should be defined as /github/workspace'
 
 outputs:
   volume:

dist/default-build-script/Assets/Editor/UnityBuilderAction/Builder.cs

@@ -29,12 +29,23 @@ namespace UnityBuilderAction
         }
       }
 
+#if UNITY_2021_2_OR_NEWER
+      // Determine subtarget
+      StandaloneBuildSubtarget buildSubtarget;
+      if (!options.TryGetValue("standaloneBuildSubtarget", out var subtargetValue) || !Enum.TryParse(subtargetValue, out buildSubtarget)) {
+        buildSubtarget = default;
+      }
+#endif
+
       // Define BuildPlayer Options
       var buildPlayerOptions = new BuildPlayerOptions {
         scenes = scenes,
         locationPathName = options["customBuildPath"],
         target = (BuildTarget) Enum.Parse(typeof(BuildTarget), options["buildTarget"]),
-        options = buildOptions
+        options = buildOptions,
+#if UNITY_2021_2_OR_NEWER
+        subtarget = (int) buildSubtarget
+#endif
       };
 
       // Set version for this build

dist/default-build-script/Assets/Editor/UnityBuilderAction/Input/AndroidSettings.cs

@@ -5,7 +5,7 @@ using System.Reflection;
 
 namespace UnityBuilderAction.Input
 {
-  public class AndroidSettings
+  public static class AndroidSettings
   {
     public static void Apply(Dictionary<string, string> options)
     {
@@ -49,8 +49,8 @@ namespace UnityBuilderAction.Input
       if (options.TryGetValue("androidExportType", out androidExportType) && !string.IsNullOrEmpty(androidExportType))
       {
         // Only exists in 2018.3 and above
-        FieldInfo buildAppBundle = typeof(EditorUserBuildSettings)
+        PropertyInfo buildAppBundle = typeof(EditorUserBuildSettings)
-              .GetField("buildAppBundle", System.Reflection.BindingFlags.Public | BindingFlags.Instance);
+              .GetProperty("buildAppBundle", BindingFlags.Public | BindingFlags.Static);
         switch (androidExportType)
         {
           case "androidStudioProject":

dist/default-build-script/Assets/Editor/UnityBuilderAction/Input/ArgumentsParser.cs

@@ -28,6 +28,7 @@ namespace UnityBuilderAction.Input
       }
 
       if (!Enum.IsDefined(typeof(BuildTarget), buildTarget)) {
+        Console.WriteLine($"{buildTarget} is not a defined {nameof(BuildTarget)}");
         EditorApplication.Exit(121);
       }
 

dist/platforms/ubuntu/entrypoint.sh

@@ -10,6 +10,7 @@ mkdir -p "$ACTIVATE_LICENSE_PATH"
 #
 # Run steps
 #
+source /steps/set_extra_git_configs.sh
 source /steps/set_gitcredential.sh
 source /steps/activate.sh
 source /steps/build.sh

dist/platforms/ubuntu/steps/build.sh

@@ -69,7 +69,7 @@ fi
 if [[ "$BUILD_TARGET" == "Android" && -n "$ANDROID_SDK_MANAGER_PARAMETERS" ]]; then
   echo "Updating Android SDK with parameters: $ANDROID_SDK_MANAGER_PARAMETERS"
   export JAVA_HOME="$(awk -F'=' '/JAVA_HOME=/{print $2}' /usr/bin/unity-editor.d/*)"
-  "$(awk -F'=' '/ANDROID_HOME=/{print $2}' /usr/bin/unity-editor.d/*)/tools/bin/sdkmanager" "$ANDROID_SDK_MANAGER_PARAMETERS"
+  "$(awk -F'=' '/ANDROID_HOME=/{print $2}' /usr/bin/unity-editor.d/*)/cmdline-tools/6.0/bin/sdkmanager" "$ANDROID_SDK_MANAGER_PARAMETERS"
   echo "Updated Android SDK."
 else
   echo "Not updating Android SDK."
@@ -119,7 +119,7 @@ echo ""
 
 unity-editor \
   -logfile /dev/stdout \
-  -quit \
+  $( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
   -customBuildName "$BUILD_NAME" \
   -projectPath "$UNITY_PROJECT_PATH" \
   -buildTarget "$BUILD_TARGET" \
@@ -153,6 +153,7 @@ fi
 
 # Make a given user owner of all artifacts
 if [[ -n "$CHOWN_FILES_TO" ]]; then
+  echo "Changing ownership of files to $CHOWN_FILES_TO for $BUILD_PATH_FULL and $UNITY_PROJECT_PATH"
   chown -R "$CHOWN_FILES_TO" "$BUILD_PATH_FULL"
   chown -R "$CHOWN_FILES_TO" "$UNITY_PROJECT_PATH"
 fi

dist/platforms/ubuntu/steps/set_extra_git_configs.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+if [ -z "${GIT_CONFIG_EXTENSIONS}" ]
+then
+  echo "GIT_CONFIG_EXTENSIONS unset skipping"
+else
+  echo "GIT_CONFIG_EXTENSIONS is set configuring extra git configs"
+
+  IFS=$'\n'
+  for config in $(echo "${GIT_CONFIG_EXTENSIONS}" | sed 's/\(.*\)=\(.*\)/"\1" "\2"/g'); do
+    if [[ $config =~ \"([^\"]+)\"\ \"([^\"]+)\" ]]; then
+      key="${BASH_REMATCH[1]}"
+      value="${BASH_REMATCH[2]}"
+    else
+      echo "Error parsing config: $config"
+      exit 1
+    fi
+    echo "Adding extra git config: \"$key\" = \"$value\""
+    git config --global --add "$key" "$value"
+  done
+  unset IFS
+
+fi
+
+echo "---------- git config --list -------------"
+git config --list
+
+echo "---------- git config --list --show-origin -------------"
+git config --list --show-origin

dist/platforms/ubuntu/steps/set_gitcredential.sh

@@ -9,8 +9,8 @@ else
 	git config --global credential.helper store
 	git config --global --replace-all url."https://token:$GIT_PRIVATE_TOKEN@github.com/".insteadOf ssh://git@github.com/
 	git config --global --add url."https://token:$GIT_PRIVATE_TOKEN@github.com/".insteadOf git@github.com
-
   git config --global --add url."https://token:$GIT_PRIVATE_TOKEN@github.com/".insteadOf "https://github.com/"
+  
   git config --global url."https://ssh:$GIT_PRIVATE_TOKEN@github.com/".insteadOf "ssh://git@github.com/"
   git config --global url."https://git:$GIT_PRIVATE_TOKEN@github.com/".insteadOf "git@github.com:"
 

dist/platforms/windows/entrypoint.ps1

@@ -1,6 +1,3 @@
-# Activate Unity
-& "c:\steps\activate.ps1"
-
 # Import any necessary registry keys, ie: location of windows 10 sdk
 # No guarantee that there will be any necessary registry keys, ie: tvOS
 Get-ChildItem -Path c:\regkeys -File | Foreach {reg import $_.fullname}
@@ -8,6 +5,12 @@ Get-ChildItem -Path c:\regkeys -File | Foreach {reg import $_.fullname}
 # Register the Visual Studio installation so Unity can find it
 regsvr32 C:\ProgramData\Microsoft\VisualStudio\Setup\x64\Microsoft.VisualStudio.Setup.Configuration.Native.dll
 
+# Setup Git Credentials
+& "c:\steps\set_gitcredential.ps1"
+
+# Activate Unity
+& "c:\steps\activate.ps1"
+
 # Build the project
 & "c:\steps\build.ps1"
 

dist/platforms/windows/set_gitcredential.ps1

@@ -0,0 +1,20 @@
+if ([string]::IsNullOrEmpty($env:GIT_PRIVATE_TOKEN)) {
+    Write-Host "GIT_PRIVATE_TOKEN unset skipping"
+}
+else {
+    Write-Host "GIT_PRIVATE_TOKEN is set configuring git credentials"
+
+    git config --global credential.helper store
+    git config --global --replace-all "url.https://token:$env:GIT_PRIVATE_TOKEN@github.com/".insteadOf "ssh://git@github.com/"
+    git config --global --add "url.https://token:$env:GIT_PRIVATE_TOKEN@github.com/".insteadOf "git@github.com"
+    git config --global --add "url.https://token:$env:GIT_PRIVATE_TOKEN@github.com/".insteadOf "https://github.com/"
+    
+    git config --global "url.https://ssh:$env:GIT_PRIVATE_TOKEN@github.com/".insteadOf "ssh://git@github.com/"
+    git config --global "url.https://git:$env:GIT_PRIVATE_TOKEN@github.com/".insteadOf "git@github.com:"
+}
+
+Write-Host "---------- git config --list -------------"
+git config --list
+
+Write-Host "---------- git config --list --show-origin -------------"
+git config --list --show-origin

dist/xhr-sync-worker.js

@@ -1,60 +0,0 @@
-"use strict";
-/* eslint-disable no-process-exit */
-const util = require("util");
-const { JSDOM } = require("../../../..");
-const { READY_STATES } = require("./xhr-utils");
-const idlUtils = require("../generated/utils");
-const tough = require("tough-cookie");
-
-const dom = new JSDOM();
-const xhr = new dom.window.XMLHttpRequest();
-const xhrImpl = idlUtils.implForWrapper(xhr);
-
-const chunks = [];
-
-process.stdin.on("data", chunk => {
-  chunks.push(chunk);
-});
-
-process.stdin.on("end", () => {
-  const buffer = Buffer.concat(chunks);
-
-  const flag = JSON.parse(buffer.toString());
-  if (flag.body && flag.body.type === "Buffer" && flag.body.data) {
-    flag.body = Buffer.from(flag.body.data);
-  }
-  if (flag.cookieJar) {
-    flag.cookieJar = tough.CookieJar.fromJSON(flag.cookieJar);
-  }
-
-  flag.synchronous = false;
-  Object.assign(xhrImpl.flag, flag);
-  const { properties } = xhrImpl;
-  xhrImpl.readyState = READY_STATES.OPENED;
-  try {
-    xhr.addEventListener("loadend", () => {
-      if (properties.error) {
-        properties.error = properties.error.stack || util.inspect(properties.error);
-      }
-      process.stdout.write(JSON.stringify({
-        responseURL: xhrImpl.responseURL,
-        status: xhrImpl.status,
-        statusText: xhrImpl.statusText,
-        properties
-      }), () => {
-        process.exit(0);
-      });
-    }, false);
-    xhr.send(flag.body);
-  } catch (error) {
-    properties.error += error.stack || util.inspect(error);
-    process.stdout.write(JSON.stringify({
-      responseURL: xhrImpl.responseURL,
-      status: xhrImpl.status,
-      statusText: xhrImpl.statusText,
-      properties
-    }), () => {
-      process.exit(0);
-    });
-  }
-});

game-ci/command-hooks/my-test-hook-post-build.yaml

@@ -1,3 +1,3 @@
-hook: after-build
+hook: after
 commands: |
   echo "after-build hook test!"

game-ci/command-hooks/my-test-hook-pre-build.yaml

@@ -1,3 +1,3 @@
-hook: before-build
+hook: before
 commands: |
   echo "before-build hook test!!"

jest.config.js

@@ -18,7 +18,6 @@ module.exports = {
   transform: {
     '^.+\\.ts$': 'ts-jest',
   },
-  autoRun: false,
 
   // Indicates whether each individual test should be reported during the run
   verbose: true,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "unity-builder",
-  "version": "2.0.0",
+  "version": "3.0.0",
   "description": "Build Unity projects for different platforms.",
   "main": "dist/index.js",
   "repository": "git@github.com:game-ci/unity-builder.git",
@@ -12,20 +12,20 @@
     "lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
     "format": "prettier --write \"src/**/*.{js,ts}\"",
     "cli": "yarn ts-node src/index.ts -m cli",
-    "gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
+    "gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
     "gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
     "aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
-    "cli-aws": "cross-env cloudRunnerCluster=aws yarn run test-cli",
+    "cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
-    "cli-k8s": "cross-env cloudRunnerCluster=k8s yarn run test-cli",
+    "cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
     "test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
     "test": "jest",
     "test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
     "test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
-    "test-i-aws": "cross-env cloudRunnerTests=true cloudRunnerCluster=aws yarn test -i -t \"cloud runner\"",
+    "test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
-    "test-i-k8s": "cross-env cloudRunnerTests=true cloudRunnerCluster=k8s yarn test -i -t \"cloud runner\""
+    "test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.x"
   },
   "dependencies": {
     "@actions/cache": "^3.1.3",
@@ -42,20 +42,22 @@
     "kubernetes-client": "^9.0.0",
     "nanoid": "^3.3.1",
     "reflect-metadata": "^0.1.13",
-    "semver": "^7.3.5",
+    "semver": "^7.5.2",
-    "unity-changeset": "^1.6.0",
+    "unity-changeset": "^2.0.0",
-    "uuid": "^8.3.2",
+    "uuid": "^9.0.0",
-    "yaml": "^1.10.2"
+    "yaml": "^2.2.2"
   },
   "devDependencies": {
-    "@arkweid/lefthook": "^0.7.7",
+    "@evilmartians/lefthook": "^1.2.9",
+    "@types/base-64": "^1.0.0",
     "@types/jest": "^27.4.1",
     "@types/node": "^17.0.23",
     "@types/semver": "^7.3.9",
+    "@types/uuid": "^9.0.0",
     "@typescript-eslint/parser": "4.8.1",
-    "@vercel/ncc": "^0.33.3",
+    "@vercel/ncc": "^0.36.1",
     "cross-env": "^7.0.3",
-    "eslint": "7.17.0",
+    "eslint": "^7.23.0",
     "eslint-config-prettier": "8.1.0",
     "eslint-plugin-github": "^4.1.1",
     "eslint-plugin-jest": "24.1.3",
@@ -63,12 +65,16 @@
     "eslint-plugin-unicorn": "28.0.2",
     "jest": "^27.5.1",
     "jest-circus": "^27.5.1",
-    "jest-fail-on-console": "^2.3.0",
+    "jest-fail-on-console": "^3.0.2",
     "js-yaml": "^4.1.0",
     "prettier": "^2.5.1",
     "ts-jest": "^27.1.3",
     "ts-node": "10.4.0",
-    "typescript": "4.1.3",
+    "typescript": "4.7.4",
     "yarn-audit-fix": "^9.3.8"
+  },
+  "volta": {
+    "node": "20.5.1",
+    "yarn": "1.22.19"
   }
 }

src/index.ts

@@ -3,6 +3,7 @@ import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output }
 import { Cli } from './model/cli/cli';
 import MacBuilder from './model/mac-builder';
 import PlatformSetup from './model/platform-setup';
+
 async function runMain() {
   try {
     if (Cli.InitCliMode()) {
@@ -18,16 +19,16 @@ async function runMain() {
     const buildParameters = await BuildParameters.create();
     const baseImage = new ImageTag(buildParameters);
 
-    if (buildParameters.cloudRunnerCluster !== 'local') {
+    if (buildParameters.providerStrategy === 'local') {
-      await CloudRunner.run(buildParameters, baseImage.toString());
-    } else {
       core.info('Building locally');
       await PlatformSetup.setup(buildParameters, actionFolder);
       if (process.platform === 'darwin') {
         MacBuilder.run(actionFolder);
       } else {
-        await Docker.run(baseImage, { workspace, actionFolder, ...buildParameters });
+        await Docker.run(baseImage.toString(), { workspace, actionFolder, ...buildParameters });
       }
+    } else {
+      await CloudRunner.run(buildParameters, baseImage.toString());
     }
 
     // Set output

src/integrity.test.ts

@@ -1,4 +1,4 @@
-import { stat } from 'fs/promises';
+import { stat } from 'node:fs/promises';
 
 describe('Integrity tests', () => {
   describe('package-lock.json', () => {

src/model/action.test.ts

@@ -1,5 +1,5 @@
-import path from 'path';
+import path from 'node:path';
-import fs from 'fs';
+import fs from 'node:fs';
 import Action from './action';
 
 describe('Action', () => {

src/model/action.ts

@@ -1,23 +1,27 @@
-import path from 'path';
+import path from 'node:path';
 
 class Action {
-  static get supportedPlatforms() {
+  static get supportedPlatforms(): string[] {
     return ['linux', 'win32', 'darwin'];
   }
 
-  static get isRunningLocally() {
+  static get isRunningLocally(): boolean {
     return process.env.RUNNER_WORKSPACE === undefined;
   }
 
-  static get isRunningFromSource() {
+  static get isRunningFromSource(): boolean {
     return path.basename(__dirname) === 'model';
   }
 
-  static get canonicalName() {
+  static get canonicalName(): string {
+    if (Action.isRunningFromSource) {
+      return path.basename(path.dirname(path.join(path.dirname(__filename), '/..')));
+    }
+
     return 'unity-builder';
   }
 
-  static get rootFolder() {
+  static get rootFolder(): string {
     if (Action.isRunningFromSource) {
       return path.dirname(path.dirname(path.dirname(__filename)));
     }
@@ -25,12 +29,12 @@ class Action {
     return path.dirname(path.dirname(__filename));
   }
 
-  static get actionFolder() {
+  static get actionFolder(): string {
     return `${Action.rootFolder}/dist`;
   }
 
-  static get workspace() {
+  static get workspace(): string {
-    return process.env.GITHUB_WORKSPACE;
+    return process.env.GITHUB_WORKSPACE!;
   }
 
   static checkCompatibility() {

src/model/android-versioning.test.ts

@@ -3,15 +3,15 @@ import AndroidVersioning from './android-versioning';
 describe('Android Versioning', () => {
   describe('versionToVersionCode', () => {
     it('defaults to 0 when versioning strategy is none', () => {
-      expect(AndroidVersioning.versionToVersionCode('none')).toBe(0);
+      expect(AndroidVersioning.versionToVersionCode('none')).toBe('0');
     });
 
     it('defaults to 1 when version is not a valid semver', () => {
-      expect(AndroidVersioning.versionToVersionCode('abcd')).toBe(1);
+      expect(AndroidVersioning.versionToVersionCode('abcd')).toBe('1');
     });
 
     it('returns a number', () => {
-      expect(AndroidVersioning.versionToVersionCode('123.456.789')).toBe(123456789);
+      expect(AndroidVersioning.versionToVersionCode('123.456.789')).toBe('123456789');
     });
 
     it('throw when generated version code is too large', () => {
@@ -21,11 +21,11 @@ describe('Android Versioning', () => {
 
   describe('determineVersionCode', () => {
     it('defaults to parsed version', () => {
-      expect(AndroidVersioning.determineVersionCode('1.2.3', '')).toBe(1002003);
+      expect(AndroidVersioning.determineVersionCode('1.2.3', '')).toBe('1002003');
     });
 
     it('use specified code', () => {
-      expect(AndroidVersioning.determineVersionCode('1.2.3', 2)).toBe(2);
+      expect(AndroidVersioning.determineVersionCode('1.2.3', '2')).toBe('2');
     });
   });
 

src/model/android-versioning.ts

@@ -2,19 +2,19 @@ import * as core from '@actions/core';
 import * as semver from 'semver';
 
 export default class AndroidVersioning {
-  static determineVersionCode(version, inputVersionCode) {
+  static determineVersionCode(version: string, inputVersionCode: string): string {
-    if (!inputVersionCode) {
+    if (inputVersionCode === '') {
       return AndroidVersioning.versionToVersionCode(version);
     }
 
     return inputVersionCode;
   }
 
-  static versionToVersionCode(version) {
+  static versionToVersionCode(version: string): string {
     if (version === 'none') {
       core.info(`Versioning strategy is set to ${version}, so android version code should not be applied.`);
 
-      return 0;
+      return '0';
     }
 
     const parsedVersion = semver.parse(version);
@@ -22,7 +22,7 @@ export default class AndroidVersioning {
     if (!parsedVersion) {
       core.warning(`Could not parse "${version}" to semver, defaulting android version code to 1`);
 
-      return 1;
+      return '1';
     }
 
     // The greatest value Google Plays allows is 2100000000.
@@ -36,10 +36,10 @@ export default class AndroidVersioning {
     }
     core.info(`Using android versionCode ${versionCode}`);
 
-    return versionCode;
+    return versionCode.toString();
   }
 
-  static determineSdkManagerParameters(targetSdkVersion) {
+  static determineSdkManagerParameters(targetSdkVersion: string) {
     const parsedVersion = Number.parseInt(targetSdkVersion.slice(-2), 10);
 
     return Number.isNaN(parsedVersion) ? '' : `platforms;android-${parsedVersion}`;

src/model/build-parameters.test.ts

@@ -33,7 +33,7 @@ describe('BuildParameters', () => {
     it('determines the unity version only once', async () => {
       jest.spyOn(UnityVersioning, 'determineUnityVersion').mockImplementation(() => '2019.2.11f1');
       await BuildParameters.create();
-      await expect(UnityVersioning.determineUnityVersion).toHaveBeenCalledTimes(1);
+      expect(UnityVersioning.determineUnityVersion).toHaveBeenCalledTimes(1);
     });
 
     it('returns the android version code with provided input', async () => {
@@ -47,13 +47,15 @@ describe('BuildParameters', () => {
     it('returns the android version code from version by default', async () => {
       const mockValue = '';
       jest.spyOn(Input, 'androidVersionCode', 'get').mockReturnValue(mockValue);
-      await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ androidVersionCode: 1003037 }));
+      await expect(BuildParameters.create()).resolves.toEqual(
+        expect.objectContaining({ androidVersionCode: '1003037' }),
+      );
     });
 
     it('determines the android sdk manager parameters only once', async () => {
       jest.spyOn(AndroidVersioning, 'determineSdkManagerParameters').mockImplementation(() => 'platforms;android-30');
       await BuildParameters.create();
-      await expect(AndroidVersioning.determineSdkManagerParameters).toHaveBeenCalledTimes(1);
+      expect(AndroidVersioning.determineSdkManagerParameters).toHaveBeenCalledTimes(1);
     });
 
     it('returns the targetPlatform', async () => {

src/model/build-parameters.ts

@@ -1,7 +1,7 @@
 import { customAlphabet } from 'nanoid';
 import AndroidVersioning from './android-versioning';
-import CloudRunnerConstants from './cloud-runner/services/cloud-runner-constants';
+import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
-import CloudRunnerBuildGuid from './cloud-runner/services/cloud-runner-guid';
+import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
 import Input from './input';
 import Platform from './platform';
 import UnityVersioning from './unity-versioning';
@@ -10,14 +10,18 @@ import { GitRepoReader } from './input-readers/git-repo';
 import { GithubCliReader } from './input-readers/github-cli';
 import { Cli } from './cli/cli';
 import GitHub from './github';
-import CloudRunnerOptions from './cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
+import CloudRunner from './cloud-runner/cloud-runner';
 
 class BuildParameters {
+  // eslint-disable-next-line no-undef
+  [key: string]: any;
+
   public editorVersion!: string;
   public customImage!: string;
   public unitySerial!: string;
   public unityLicensingServer!: string;
-  public runnerTempPath: string | undefined;
+  public runnerTempPath!: string;
   public targetPlatform!: string;
   public projectPath!: string;
   public buildName!: string;
@@ -25,6 +29,7 @@ class BuildParameters {
   public buildFile!: string;
   public buildMethod!: string;
   public buildVersion!: string;
+  public manualExit!: boolean;
   public androidVersionCode!: string;
   public androidKeystoreName!: string;
   public androidKeystoreBase64!: string;
@@ -38,24 +43,24 @@ class BuildParameters {
 
   public customParameters!: string;
   public sshAgent!: string;
-  public cloudRunnerCluster!: string;
+  public sshPublicKeysDirectoryPath!: string;
-  public awsBaseStackName!: string;
+  public providerStrategy!: string;
   public gitPrivateToken!: string;
   public awsStackName!: string;
   public kubeConfig!: string;
-  public cloudRunnerMemory!: string;
+  public containerMemory!: string;
-  public cloudRunnerCpu!: string;
+  public containerCpu!: string;
   public kubeVolumeSize!: string;
   public kubeVolume!: string;
   public kubeStorageClass!: string;
   public chownFilesTo!: string;
-  public customJobHooks!: string;
+  public commandHooks!: string;
-  public readInputFromOverrideList!: string;
+  public pullInputList!: string[];
-  public readInputOverrideCommand!: string;
+  public inputPullCommand!: string;
   public cacheKey!: string;
 
-  public postBuildSteps!: string;
+  public postBuildContainerHooks!: string;
-  public preBuildSteps!: string;
+  public preBuildContainerHooks!: string;
   public customJob!: string;
   public runNumber!: string;
   public branch!: string;
@@ -64,18 +69,26 @@ class BuildParameters {
   public logId!: string;
   public buildGuid!: string;
   public cloudRunnerBranch!: string;
-  public cloudRunnerDebug!: boolean;
+  public cloudRunnerDebug!: boolean | undefined;
-  public cloudRunnerBuilderPlatform!: string | undefined;
+  public buildPlatform!: string | undefined;
   public isCliMode!: boolean;
-  public retainWorkspace!: boolean;
   public maxRetainedWorkspaces!: number;
-  public useSharedLargePackages!: boolean;
+  public useLargePackages!: boolean;
-  public useLz4Compression!: boolean;
+  public useCompressionStrategy!: boolean;
-  public garbageCollectionMaxAge!: number;
+  public garbageMaxAge!: number;
-  public constantGarbageCollection!: boolean;
   public githubChecks!: boolean;
+  public asyncWorkflow!: boolean;
+  public githubCheckId!: string;
+  public finalHooks!: string[];
+  public skipLfs!: boolean;
+  public skipCache!: boolean;
   public cacheUnityInstallationOnMac!: boolean;
   public unityHubVersionOnMac!: string;
+  public dockerWorkspacePath!: string;
+
+  public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
+    return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
+  }
 
   static async create(): Promise<BuildParameters> {
     const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
@@ -98,21 +111,19 @@ class BuildParameters {
       }
     }
 
-    // Todo - Don't use process.env directly, that's what the input model class is for.
-    // ---
     let unitySerial = '';
     if (Input.unityLicensingServer === '') {
-      if (!process.env.UNITY_SERIAL && GitHub.githubInputEnabled) {
+      if (!Input.unitySerial && GitHub.githubInputEnabled) {
         // No serial was present, so it is a personal license that we need to convert
-        if (!process.env.UNITY_LICENSE) {
+        if (!Input.unityLicense) {
           throw new Error(`Missing Unity License File and no Serial was found. If this
                             is a personal license, make sure to follow the activation
                             steps and set the UNITY_LICENSE GitHub secret or enter a Unity
                             serial number inside the UNITY_SERIAL GitHub secret.`);
         }
-        unitySerial = this.getSerialFromLicenseFile(process.env.UNITY_LICENSE);
+        unitySerial = this.getSerialFromLicenseFile(Input.unityLicense);
       } else {
-        unitySerial = process.env.UNITY_SERIAL!;
+        unitySerial = Input.unitySerial!;
       }
     }
 
@@ -121,7 +132,7 @@ class BuildParameters {
       customImage: Input.customImage,
       unitySerial,
       unityLicensingServer: Input.unityLicensingServer,
-      runnerTempPath: process.env.RUNNER_TEMP,
+      runnerTempPath: Input.runnerTempPath,
       targetPlatform: Input.targetPlatform,
       projectPath: Input.projectPath,
       buildName: Input.buildName,
@@ -129,6 +140,7 @@ class BuildParameters {
       buildFile,
       buildMethod: Input.buildMethod,
       buildVersion,
+      manualExit: Input.manualExit,
       androidVersionCode,
       androidKeystoreName: Input.androidKeystoreName,
       androidKeystoreBase64: Input.androidKeystoreBase64,
@@ -141,18 +153,18 @@ class BuildParameters {
       androidSymbolType: androidSymbolExportType,
       customParameters: Input.customParameters,
       sshAgent: Input.sshAgent,
+      sshPublicKeysDirectoryPath: Input.sshPublicKeysDirectoryPath,
       gitPrivateToken: Input.gitPrivateToken || (await GithubCliReader.GetGitHubAuthToken()),
       chownFilesTo: Input.chownFilesTo,
-      cloudRunnerCluster: CloudRunnerOptions.cloudRunnerCluster,
+      providerStrategy: CloudRunnerOptions.providerStrategy,
-      cloudRunnerBuilderPlatform: CloudRunnerOptions.cloudRunnerBuilderPlatform,
+      buildPlatform: CloudRunnerOptions.buildPlatform,
-      awsBaseStackName: CloudRunnerOptions.awsBaseStackName,
       kubeConfig: CloudRunnerOptions.kubeConfig,
-      cloudRunnerMemory: CloudRunnerOptions.cloudRunnerMemory,
+      containerMemory: CloudRunnerOptions.containerMemory,
-      cloudRunnerCpu: CloudRunnerOptions.cloudRunnerCpu,
+      containerCpu: CloudRunnerOptions.containerCpu,
       kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
       kubeVolume: CloudRunnerOptions.kubeVolume,
-      postBuildSteps: CloudRunnerOptions.postBuildSteps,
+      postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
-      preBuildSteps: CloudRunnerOptions.preBuildSteps,
+      preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
       customJob: CloudRunnerOptions.customJob,
       runNumber: Input.runNumber,
       branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
@@ -160,24 +172,28 @@ class BuildParameters {
       cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
       githubRepo: Input.githubRepo || (await GitRepoReader.GetRemote()) || 'game-ci/unity-builder',
       isCliMode: Cli.isCliMode,
-      awsStackName: CloudRunnerOptions.awsBaseStackName,
+      awsStackName: CloudRunnerOptions.awsStackName,
       gitSha: Input.gitSha,
       logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
       buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
-      customJobHooks: CloudRunnerOptions.customJobHooks(),
+      commandHooks: CloudRunnerOptions.commandHooks,
-      readInputOverrideCommand: CloudRunnerOptions.readInputOverrideCommand(),
+      inputPullCommand: CloudRunnerOptions.inputPullCommand,
-      readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
+      pullInputList: CloudRunnerOptions.pullInputList,
       kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
       cacheKey: CloudRunnerOptions.cacheKey,
-      retainWorkspace: CloudRunnerOptions.retainWorkspaces,
+      maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
-      useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
+      useLargePackages: CloudRunnerOptions.useLargePackages,
-      useLz4Compression: CloudRunnerOptions.useLz4Compression,
+      useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
-      maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
+      garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
-      constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection,
-      garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
       githubChecks: CloudRunnerOptions.githubChecks,
+      asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
+      githubCheckId: CloudRunnerOptions.githubCheckId,
+      finalHooks: CloudRunnerOptions.finalHooks,
+      skipLfs: CloudRunnerOptions.skipLfs,
+      skipCache: CloudRunnerOptions.skipCache,
       cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
       unityHubVersionOnMac: Input.unityHubVersionOnMac,
+      dockerWorkspacePath: Input.dockerWorkspacePath,
     };
   }
 

src/model/cache.ts

@@ -1,5 +1,5 @@
 import * as core from '@actions/core';
-import fs from 'fs';
+import fs from 'node:fs';
 import Action from './action';
 import Project from './project';
 

src/model/cli/cli-functions-repository.ts

@@ -16,7 +16,7 @@ export class CliFunctionsRepository {
     });
   }
 
-  public static GetCliFunctions(key) {
+  public static GetCliFunctions(key: any) {
     const results = CliFunctionsRepository.targets.find((x) => x.key === key);
     if (results === undefined || results.length === 0) {
       throw new Error(`no CLI mode found for ${key}`);

src/model/cli/cli.ts

@@ -2,24 +2,25 @@ import { Command } from 'commander-ts';
 import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
 import * as core from '@actions/core';
 import { ActionYamlReader } from '../input-readers/action-yaml';
-import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
+import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
-import CloudRunnerQueryOverride from '../cloud-runner/services/cloud-runner-query-override';
+import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
 import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
 import { Caching } from '../cloud-runner/remote-client/caching';
-import { LfsHashing } from '../cloud-runner/services/lfs-hashing';
+import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
 import { RemoteClient } from '../cloud-runner/remote-client';
-import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-options-reader';
+import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
 import GitHub from '../github';
-import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer';
+import { CloudRunnerFolders } from '../cloud-runner/options/cloud-runner-folders';
-import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
+import { OptionValues } from 'commander';
+import { InputKey } from '../input';
 
 export class Cli {
-  public static options;
+  public static options: OptionValues | undefined;
   static get isCliMode() {
     return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== '';
   }
-  public static query(key, alternativeKey) {
+  public static query(key: string, alternativeKey: string) {
     if (Cli.options && Cli.options[key] !== undefined) {
       return Cli.options[key];
     }
@@ -61,22 +62,24 @@ export class Cli {
 
   static async RunCli(): Promise<void> {
     GitHub.githubInputEnabled = false;
-    if (Cli.options['populateOverride'] === `true`) {
+    if (Cli.options!['populateOverride'] === `true`) {
       await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
     }
-    if (Cli.options['logInput']) {
+    if (Cli.options!['logInput']) {
       Cli.logInput();
     }
-    const results = CliFunctionsRepository.GetCliFunctions(Cli.options.mode);
+    const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
     CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
-    Cli.options.versioning = 'None';
+    Cli.options!.versioning = 'None';
 
-    const buildParameter = TaskParameterSerializer.readBuildParameterFromEnvironment();
+    CloudRunner.buildParameters = await BuildParameters.create();
+    CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
     CloudRunnerLogger.log(`Build Params:
-      ${JSON.stringify(buildParameter, undefined, 4)}
+      ${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
     `);
-    CloudRunner.buildParameters = buildParameter;
+    CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
-    CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
+    CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
+    await CloudRunner.setup(CloudRunner.buildParameters);
 
     return await results.target[results.propertyKey](Cli.options);
   }
@@ -88,14 +91,15 @@ export class Cli {
     const properties = CloudRunnerOptionsReader.GetProperties();
     for (const element of properties) {
       if (
-        Input[element] !== undefined &&
+        element in Input &&
-        Input[element] !== '' &&
+        Input[element as InputKey] !== undefined &&
-        typeof Input[element] !== `function` &&
+        Input[element as InputKey] !== '' &&
+        typeof Input[element as InputKey] !== `function` &&
         element !== 'length' &&
         element !== 'cliOptions' &&
         element !== 'prototype'
       ) {
-        core.info(`${element} ${Input[element]}`);
+        core.info(`${element} ${Input[element as InputKey]}`);
       }
     }
     core.info(`\n`);
@@ -113,12 +117,16 @@ export class Cli {
   public static async asyncronousWorkflow(): Promise<string> {
     const buildParameter = await BuildParameters.create();
     const baseImage = new ImageTag(buildParameter);
+    await CloudRunner.setup(buildParameter);
 
     return await CloudRunner.run(buildParameter, baseImage.toString());
   }
 
   @CliFunction(`checks-update`, `runs a cloud runner build`)
   public static async checksUpdate() {
+    const buildParameter = await BuildParameters.create();
+
+    await CloudRunner.setup(buildParameter);
     const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
     core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
     if (input.mode === `create`) {
@@ -182,7 +190,7 @@ export class Cli {
       `build-${CloudRunner.buildParameters.buildGuid}`,
     );
 
-    if (!CloudRunner.buildParameters.retainWorkspace) {
+    if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
       await CloudRunnerSystem.Run(
         `rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
       );
@@ -190,21 +198,6 @@ export class Cli {
 
     await RemoteClient.runCustomHookFiles(`after-build`);
 
-    const parameters = await BuildParameters.create();
-    CloudRunner.setup(parameters);
-    if (parameters.constantGarbageCollection) {
-      await CloudRunnerSystem.Run(
-        `find /${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.buildVolumeFolder)}/ -name '*.*' -mmin +${
-          parameters.garbageCollectionMaxAge * 60
-        } -delete`,
-      );
-      await CloudRunnerSystem.Run(
-        `find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${
-          parameters.garbageCollectionMaxAge * 60
-        } -delete`,
-      );
-    }
-
     return new Promise((result) => result(``));
   }
 }

src/model/cloud-runner/cloud-runner-options.ts

@@ -1,290 +0,0 @@
-import { Cli } from '../cli/cli';
-import CloudRunnerQueryOverride from './services/cloud-runner-query-override';
-import GitHub from '../github';
-const core = require('@actions/core');
-
-class CloudRunnerOptions {
-  // ### ### ###
-  // Input Handling
-  // ### ### ###
-  public static getInput(query) {
-    if (GitHub.githubInputEnabled) {
-      const coreInput = core.getInput(query);
-      if (coreInput && coreInput !== '') {
-        return coreInput;
-      }
-    }
-    const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
-
-    // Query input sources
-    if (Cli.query(query, alternativeQuery)) {
-      return Cli.query(query, alternativeQuery);
-    }
-
-    if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
-      return CloudRunnerQueryOverride.query(query, alternativeQuery);
-    }
-
-    if (process.env[query] !== undefined) {
-      return process.env[query];
-    }
-
-    if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
-      return process.env[alternativeQuery];
-    }
-
-    return;
-  }
-
-  public static ToEnvVarFormat(input: string) {
-    if (input.toUpperCase() === input) {
-      return input;
-    }
-
-    return input
-      .replace(/([A-Z])/g, ' $1')
-      .trim()
-      .toUpperCase()
-      .replace(/ /g, '_');
-  }
-
-  // ### ### ###
-  // Provider parameters
-  // ### ### ###
-
-  static get region(): string {
-    return CloudRunnerOptions.getInput('region') || 'eu-west-2';
-  }
-
-  // ### ### ###
-  // GitHub  parameters
-  // ### ### ###
-  static get githubChecks(): boolean {
-    return CloudRunnerOptions.getInput('githubChecks') || false;
-  }
-
-  static get githubOwner() {
-    return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo.split(`/`)[0] || false;
-  }
-
-  static get githubRepoName() {
-    return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo.split(`/`)[1] || false;
-  }
-
-  // ### ### ###
-  // Git syncronization parameters
-  // ### ### ###
-
-  static get githubRepo() {
-    return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
-  }
-  static get branch() {
-    if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
-      return CloudRunnerOptions.getInput(`GITHUB_REF`).replace('refs/', '').replace(`head/`, '').replace(`heads/`, '');
-    } else if (CloudRunnerOptions.getInput('branch')) {
-      return CloudRunnerOptions.getInput('branch');
-    } else {
-      return '';
-    }
-  }
-
-  static get gitSha() {
-    if (CloudRunnerOptions.getInput(`GITHUB_SHA`)) {
-      return CloudRunnerOptions.getInput(`GITHUB_SHA`);
-    } else if (CloudRunnerOptions.getInput(`GitSHA`)) {
-      return CloudRunnerOptions.getInput(`GitSHA`);
-    }
-  }
-
-  // ### ### ###
-  // Cloud Runner parameters
-  // ### ### ###
-
-  static get cloudRunnerBuilderPlatform() {
-    const input = CloudRunnerOptions.getInput('cloudRunnerBuilderPlatform');
-    if (input) {
-      return input;
-    }
-    if (CloudRunnerOptions.cloudRunnerCluster !== 'local') {
-      return 'linux';
-    }
-
-    return;
-  }
-
-  static get cloudRunnerBranch() {
-    return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
-  }
-
-  static get cloudRunnerCluster() {
-    if (Cli.isCliMode) {
-      return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'aws';
-    }
-
-    return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'local';
-  }
-
-  static get cloudRunnerCpu() {
-    return CloudRunnerOptions.getInput('cloudRunnerCpu');
-  }
-
-  static get cloudRunnerMemory() {
-    return CloudRunnerOptions.getInput('cloudRunnerMemory');
-  }
-
-  static get customJob() {
-    return CloudRunnerOptions.getInput('customJob') || '';
-  }
-
-  // ### ### ###
-  // Custom commands from files parameters
-  // ### ### ###
-
-  static get customStepFiles() {
-    return CloudRunnerOptions.getInput('customStepFiles')?.split(`,`) || [];
-  }
-
-  static get customHookFiles() {
-    return CloudRunnerOptions.getInput('customHookFiles')?.split(`,`) || [];
-  }
-
-  // ### ### ###
-  // Custom commands from yaml parameters
-  // ### ### ###
-
-  static customJobHooks() {
-    return CloudRunnerOptions.getInput('customJobHooks') || '';
-  }
-
-  static get postBuildSteps() {
-    return CloudRunnerOptions.getInput('postBuildSteps') || '';
-  }
-
-  static get preBuildSteps() {
-    return CloudRunnerOptions.getInput('preBuildSteps') || '';
-  }
-
-  // ### ### ###
-  // Input override handling
-  // ### ### ###
-
-  static readInputFromOverrideList() {
-    return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
-  }
-
-  static readInputOverrideCommand() {
-    const value = CloudRunnerOptions.getInput('readInputOverrideCommand');
-
-    if (value === 'gcp-secret-manager') {
-      return 'gcloud secrets versions access 1 --secret="{0}"';
-    } else if (value === 'aws-secret-manager') {
-      return 'aws secretsmanager get-secret-value --secret-id {0}';
-    }
-
-    return value || '';
-  }
-
-  // ### ### ###
-  // Aws
-  // ### ### ###
-
-  static get awsBaseStackName() {
-    return CloudRunnerOptions.getInput('awsBaseStackName') || 'game-ci';
-  }
-
-  // ### ### ###
-  // K8s
-  // ### ### ###
-
-  static get kubeConfig() {
-    return CloudRunnerOptions.getInput('kubeConfig') || '';
-  }
-
-  static get kubeVolume() {
-    return CloudRunnerOptions.getInput('kubeVolume') || '';
-  }
-
-  static get kubeVolumeSize() {
-    return CloudRunnerOptions.getInput('kubeVolumeSize') || '5Gi';
-  }
-
-  static get kubeStorageClass(): string {
-    return CloudRunnerOptions.getInput('kubeStorageClass') || '';
-  }
-
-  // ### ### ###
-  // Caching
-  // ### ### ###
-
-  static get cacheKey(): string {
-    return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
-  }
-
-  // ### ### ###
-  // Utility Parameters
-  // ### ### ###
-
-  static get cloudRunnerDebug(): boolean {
-    return CloudRunnerOptions.getInput(`cloudRunnerTests`) || CloudRunnerOptions.getInput(`cloudRunnerDebug`) || false;
-  }
-  static get cloudRunnerDebugTree(): boolean {
-    return CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) || false;
-  }
-  static get cloudRunnerDebugEnv(): boolean {
-    return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) || false;
-  }
-
-  static get watchCloudRunnerToEnd(): boolean {
-    if (CloudRunnerOptions.asyncCloudRunner) {
-      return false;
-    }
-
-    return CloudRunnerOptions.getInput(`watchToEnd`) || true;
-  }
-
-  static get asyncCloudRunner(): boolean {
-    return (CloudRunnerOptions.getInput('asyncCloudRunner') || `false`) === `true` || false;
-  }
-
-  public static get useSharedLargePackages(): boolean {
-    return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') === 'true';
-  }
-
-  public static get useSharedBuilder(): boolean {
-    return (CloudRunnerOptions.getInput(`useSharedBuilder`) || 'true') === 'true';
-  }
-
-  public static get useLz4Compression(): boolean {
-    return (CloudRunnerOptions.getInput(`useLz4Compression`) || 'false') === 'true';
-  }
-
-  public static get useCleanupCron(): boolean {
-    return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
-  }
-
-  // ### ### ###
-  // Retained Workspace
-  // ### ### ###
-
-  public static get retainWorkspaces(): boolean {
-    return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
-  }
-
-  static get maxRetainedWorkspaces(): number {
-    return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
-  }
-
-  // ### ### ###
-  // Garbage Collection
-  // ### ### ###
-
-  static get constantGarbageCollection(): boolean {
-    return CloudRunnerOptions.getInput(`constantGarbageCollection`) || true;
-  }
-
-  static get garbageCollectionMaxAge(): number {
-    return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
-  }
-}
-
-export default CloudRunnerOptions;

src/model/cloud-runner/cloud-runner.ts

@@ -1,33 +1,42 @@
 import AwsBuildPlatform from './providers/aws';
 import { BuildParameters, Input } from '..';
 import Kubernetes from './providers/k8s';
-import CloudRunnerLogger from './services/cloud-runner-logger';
+import CloudRunnerLogger from './services/core/cloud-runner-logger';
-import { CloudRunnerStepState } from './cloud-runner-step-state';
+import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
 import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
 import { CloudRunnerError } from './error/cloud-runner-error';
-import { TaskParameterSerializer } from './services/task-parameter-serializer';
+import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
 import * as core from '@actions/core';
-import CloudRunnerSecret from './services/cloud-runner-secret';
+import CloudRunnerSecret from './options/cloud-runner-secret';
 import { ProviderInterface } from './providers/provider-interface';
-import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
 import TestCloudRunner from './providers/test';
 import LocalCloudRunner from './providers/local';
 import LocalDockerCloudRunner from './providers/docker';
 import GitHub from '../github';
-import SharedWorkspaceLocking from './services/shared-workspace-locking';
+import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
+import { FollowLogStreamService } from './services/core/follow-log-stream-service';
 
 class CloudRunner {
   public static Provider: ProviderInterface;
   public static buildParameters: BuildParameters;
   private static defaultSecrets: CloudRunnerSecret[];
   private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
-  static lockedWorkspace: string | undefined;
+  static lockedWorkspace: string = ``;
   public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
-  public static githubCheckId;
+  public static get isCloudRunnerEnvironment() {
-  public static setup(buildParameters: BuildParameters) {
+    return process.env[`GITHUB_ACTIONS`] !== `true`;
+  }
+  public static get isCloudRunnerAsyncEnvironment() {
+    return process.env[`ASYNC_WORKFLOW`] === `true`;
+  }
+  public static async setup(buildParameters: BuildParameters) {
     CloudRunnerLogger.setup();
     CloudRunnerLogger.log(`Setting up cloud runner`);
     CloudRunner.buildParameters = buildParameters;
+    if (CloudRunner.buildParameters.githubCheckId === ``) {
+      CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
+    }
     CloudRunner.setupSelectedBuildPlatform();
     CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
     CloudRunner.cloudRunnerEnvironmentVariables =
@@ -45,15 +54,16 @@ class CloudRunner {
       core.setOutput(
         Input.ToEnvVarFormat(`buildArtifact`),
         `build-${CloudRunner.buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       );
     }
+    FollowLogStreamService.Reset();
   }
 
   private static setupSelectedBuildPlatform() {
-    CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.cloudRunnerCluster}`);
+    CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
-    switch (CloudRunner.buildParameters.cloudRunnerCluster) {
+    switch (CloudRunner.buildParameters.providerStrategy) {
       case 'k8s':
         CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
         break;
@@ -73,14 +83,20 @@ class CloudRunner {
   }
 
   static async run(buildParameters: BuildParameters, baseImage: string) {
-    CloudRunner.setup(buildParameters);
+    await CloudRunner.setup(buildParameters);
+    if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
+    await CloudRunner.Provider.setupWorkflow(
+      CloudRunner.buildParameters.buildGuid,
+      CloudRunner.buildParameters,
+      CloudRunner.buildParameters.branch,
+      CloudRunner.defaultSecrets,
+    );
+    if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
     try {
-      CloudRunner.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
+      if (buildParameters.maxRetainedWorkspaces > 0) {
+        CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
 
-      if (buildParameters.retainWorkspace) {
+        const result = await SharedWorkspaceLocking.GetLockedWorkspace(
-        CloudRunner.lockedWorkspace = `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
-
-        const result = await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
           CloudRunner.lockedWorkspace,
           CloudRunner.buildParameters.buildGuid,
           CloudRunner.buildParameters,
@@ -94,21 +110,21 @@ class CloudRunner {
           ];
         } else {
           CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
-          buildParameters.retainWorkspace = false;
+          buildParameters.maxRetainedWorkspaces = 0;
-          CloudRunner.lockedWorkspace = undefined;
+          CloudRunner.lockedWorkspace = ``;
         }
       }
-      if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
+      const content = { ...CloudRunner.buildParameters };
-      await CloudRunner.Provider.setupWorkflow(
+      content.gitPrivateToken = ``;
-        CloudRunner.buildParameters.buildGuid,
+      content.unitySerial = ``;
-        CloudRunner.buildParameters,
+      const jsonContent = JSON.stringify(content, undefined, 4);
-        CloudRunner.buildParameters.branch,
+      await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
-        CloudRunner.defaultSecrets,
-      );
-      if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
-      await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid);
       const output = await new WorkflowCompositionRoot().run(
-        new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
+        new CloudRunnerStepParameters(
+          baseImage,
+          CloudRunner.cloudRunnerEnvironmentVariables,
+          CloudRunner.defaultSecrets,
+        ),
       );
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Cleanup shared cloud runner resources');
       await CloudRunner.Provider.cleanupWorkflow(
@@ -121,22 +137,40 @@ class CloudRunner {
       if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
       await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
 
-      if (CloudRunner.buildParameters.retainWorkspace) {
+      if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
+        const workspace = CloudRunner.lockedWorkspace || ``;
         await SharedWorkspaceLocking.ReleaseWorkspace(
-          CloudRunner.lockedWorkspace || ``,
+          workspace,
           CloudRunner.buildParameters.buildGuid,
           CloudRunner.buildParameters,
         );
-        CloudRunner.lockedWorkspace = undefined;
+        const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
+        if (isLocked) {
+          throw new Error(
+            `still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
+              workspace,
+              buildParameters,
+            )}`,
+          );
+        }
+        CloudRunner.lockedWorkspace = ``;
       }
 
+      await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
+
       if (buildParameters.constantGarbageCollection) {
-        CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
+        CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
       }
 
       return output;
-    } catch (error) {
+    } catch (error: any) {
-      await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, error, `failure`, `completed`);
+      CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
+      await GitHub.updateGitHubCheck(
+        CloudRunner.buildParameters.buildGuid,
+        `Failed - Error ${error?.message || error}`,
+        `failure`,
+        `completed`,
+      );
       if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
       await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
       throw error;

src/model/cloud-runner/error/cloud-runner-error.ts

@@ -1,7 +1,7 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 import * as core from '@actions/core';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
 import BuildParameters from '../../build-parameters';
 
 export class CloudRunnerError {

src/model/cloud-runner/options/cloud-runner-folders.ts

@@ -1,6 +1,7 @@
-import path from 'path';
+import path from 'node:path';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
-import CloudRunner from './../cloud-runner';
+import CloudRunner from '../cloud-runner';
+import BuildParameters from '../../build-parameters';
 
 export class CloudRunnerFolders {
   public static readonly repositoryFolder = 'repo';
@@ -12,7 +13,7 @@ export class CloudRunnerFolders {
   // Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
 
   public static get uniqueCloudRunnerJobFolderAbsolute(): string {
-    return CloudRunner.buildParameters && CloudRunner.buildParameters.retainWorkspace && CloudRunner.lockedWorkspace
+    return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
       ? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
       : path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
   }

src/model/cloud-runner/options/cloud-runner-options-reader.ts

@@ -1,5 +1,5 @@
 import Input from '../../input';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
 
 class CloudRunnerOptionsReader {
   static GetProperties() {

src/model/cloud-runner/options/cloud-runner-options.ts

@@ -0,0 +1,283 @@
+import { Cli } from '../../cli/cli';
+import CloudRunnerQueryOverride from './cloud-runner-query-override';
+import GitHub from '../../github';
+import * as core from '@actions/core';
+
+class CloudRunnerOptions {
+  // ### ### ###
+  // Input Handling
+  // ### ### ###
+  public static getInput(query: string): string | undefined {
+    if (GitHub.githubInputEnabled) {
+      const coreInput = core.getInput(query);
+      if (coreInput && coreInput !== '') {
+        return coreInput;
+      }
+    }
+    const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
+
+    // Query input sources
+    if (Cli.query(query, alternativeQuery)) {
+      return Cli.query(query, alternativeQuery);
+    }
+
+    if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
+      return CloudRunnerQueryOverride.query(query, alternativeQuery);
+    }
+
+    if (process.env[query] !== undefined) {
+      return process.env[query];
+    }
+
+    if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
+      return process.env[alternativeQuery];
+    }
+  }
+
+  public static ToEnvVarFormat(input: string): string {
+    if (input.toUpperCase() === input) {
+      return input;
+    }
+
+    return input
+      .replace(/([A-Z])/g, ' $1')
+      .trim()
+      .toUpperCase()
+      .replace(/ /g, '_');
+  }
+
+  // ### ### ###
+  // Provider parameters
+  // ### ### ###
+
+  static get region(): string {
+    return CloudRunnerOptions.getInput('region') || 'eu-west-2';
+  }
+
+  // ### ### ###
+  // GitHub  parameters
+  // ### ### ###
+  static get githubChecks(): boolean {
+    const value = CloudRunnerOptions.getInput('githubChecks');
+
+    return value === `true` || false;
+  }
+  static get githubCheckId(): string {
+    return CloudRunnerOptions.getInput('githubCheckId') || ``;
+  }
+
+  static get githubOwner(): string {
+    return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
+  }
+
+  static get githubRepoName(): string {
+    return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
+  }
+
+  static get finalHooks(): string[] {
+    return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
+  }
+
+  // ### ### ###
+  // Git syncronization parameters
+  // ### ### ###
+
+  static get githubRepo(): string | undefined {
+    return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
+  }
+  static get branch(): string {
+    if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
+      return (
+        CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
+      );
+    } else if (CloudRunnerOptions.getInput('branch')) {
+      return CloudRunnerOptions.getInput('branch') || ``;
+    } else {
+      return '';
+    }
+  }
+
+  // ### ### ###
+  // Cloud Runner parameters
+  // ### ### ###
+
+  static get buildPlatform(): string {
+    const input = CloudRunnerOptions.getInput('buildPlatform');
+    if (input) {
+      return input;
+    }
+    if (CloudRunnerOptions.providerStrategy !== 'local') {
+      return 'linux';
+    }
+
+    return ``;
+  }
+
+  static get cloudRunnerBranch(): string {
+    return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
+  }
+
+  static get providerStrategy(): string {
+    const provider =
+      CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
+    if (Cli.isCliMode) {
+      return provider || 'aws';
+    }
+
+    return provider || 'local';
+  }
+
+  static get containerCpu(): string {
+    return CloudRunnerOptions.getInput('containerCpu') || `1024`;
+  }
+
+  static get containerMemory(): string {
+    return CloudRunnerOptions.getInput('containerMemory') || `3072`;
+  }
+
+  static get customJob(): string {
+    return CloudRunnerOptions.getInput('customJob') || '';
+  }
+
+  // ### ### ###
+  // Custom commands from files parameters
+  // ### ### ###
+
+  static get containerHookFiles(): string[] {
+    return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
+  }
+
+  static get commandHookFiles(): string[] {
+    return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
+  }
+
+  // ### ### ###
+  // Custom commands from yaml parameters
+  // ### ### ###
+
+  static get commandHooks(): string {
+    return CloudRunnerOptions.getInput('commandHooks') || '';
+  }
+
+  static get postBuildContainerHooks(): string {
+    return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
+  }
+
+  static get preBuildContainerHooks(): string {
+    return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
+  }
+
+  // ### ### ###
+  // Input override handling
+  // ### ### ###
+
+  static get pullInputList(): string[] {
+    return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
+  }
+
+  static get inputPullCommand(): string {
+    const value = CloudRunnerOptions.getInput('inputPullCommand');
+
+    if (value === 'gcp-secret-manager') {
+      return 'gcloud secrets versions access 1 --secret="{0}"';
+    } else if (value === 'aws-secret-manager') {
+      return 'aws secretsmanager get-secret-value --secret-id {0}';
+    }
+
+    return value || '';
+  }
+
+  // ### ### ###
+  // Aws
+  // ### ### ###
+
+  static get awsStackName() {
+    return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
+  }
+
+  // ### ### ###
+  // K8s
+  // ### ### ###
+
+  static get kubeConfig(): string {
+    return CloudRunnerOptions.getInput('kubeConfig') || '';
+  }
+
+  static get kubeVolume(): string {
+    return CloudRunnerOptions.getInput('kubeVolume') || '';
+  }
+
+  static get kubeVolumeSize(): string {
+    return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
+  }
+
+  static get kubeStorageClass(): string {
+    return CloudRunnerOptions.getInput('kubeStorageClass') || '';
+  }
+
+  // ### ### ###
+  // Caching
+  // ### ### ###
+
+  static get cacheKey(): string {
+    return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
+  }
+
+  // ### ### ###
+  // Utility Parameters
+  // ### ### ###
+
+  static get cloudRunnerDebug(): boolean {
+    return (
+      CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
+      false
+    );
+  }
+  static get skipLfs(): boolean {
+    return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
+  }
+  static get skipCache(): boolean {
+    return CloudRunnerOptions.getInput(`skipCache`) === `true`;
+  }
+
+  public static get asyncCloudRunner(): boolean {
+    return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
+  }
+
+  public static get useLargePackages(): boolean {
+    return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
+  }
+
+  public static get useSharedBuilder(): boolean {
+    return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
+  }
+
+  public static get useCompressionStrategy(): boolean {
+    return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
+  }
+
+  public static get useCleanupCron(): boolean {
+    return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
+  }
+
+  // ### ### ###
+  // Retained Workspace
+  // ### ### ###
+
+  public static get maxRetainedWorkspaces(): string {
+    return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
+  }
+
+  // ### ### ###
+  // Garbage Collection
+  // ### ### ###
+
+  static get garbageMaxAge(): number {
+    return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
+  }
+}
+
+export default CloudRunnerOptions;

src/model/cloud-runner/options/cloud-runner-query-override.ts

@@ -1,8 +1,8 @@
 import Input from '../../input';
 import { GenericInputReader } from '../../input-readers/generic-input-reader';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
 
-const formatFunction = (value, arguments_) => {
+const formatFunction = (value: string, arguments_: any[]) => {
   for (const element of arguments_) {
     value = value.replace(`{${element.key}}`, element.value);
   }
@@ -11,11 +11,11 @@ const formatFunction = (value, arguments_) => {
 };
 
 class CloudRunnerQueryOverride {
-  static queryOverrides: any;
+  static queryOverrides: { [key: string]: string } | undefined;
 
   // TODO accept premade secret sources or custom secret source definition yamls
 
-  public static query(key, alternativeKey) {
+  public static query(key: string, alternativeKey: string) {
     if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
       return CloudRunnerQueryOverride.queryOverrides[key];
     }
@@ -30,12 +30,12 @@ class CloudRunnerQueryOverride {
     return;
   }
 
-  private static shouldUseOverride(query) {
+  private static shouldUseOverride(query: string) {
-    if (CloudRunnerOptions.readInputOverrideCommand() !== '') {
+    if (CloudRunnerOptions.inputPullCommand !== '') {
-      if (CloudRunnerOptions.readInputFromOverrideList() !== '') {
+      if (CloudRunnerOptions.pullInputList.length > 0) {
         const doesInclude =
-          CloudRunnerOptions.readInputFromOverrideList().split(',').includes(query) ||
+          CloudRunnerOptions.pullInputList.includes(query) ||
-          CloudRunnerOptions.readInputFromOverrideList().split(',').includes(Input.ToEnvVarFormat(query));
+          CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
 
         return doesInclude ? true : false;
       } else {
@@ -44,19 +44,19 @@ class CloudRunnerQueryOverride {
     }
   }
 
-  private static async queryOverride(query) {
+  private static async queryOverride(query: string) {
     if (!this.shouldUseOverride(query)) {
       throw new Error(`Should not be trying to run override query on ${query}`);
     }
 
     return await GenericInputReader.Run(
-      formatFunction(CloudRunnerOptions.readInputOverrideCommand(), [{ key: 0, value: query }]),
+      formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
     );
   }
 
   public static async PopulateQueryOverrideInput() {
-    const queries = CloudRunnerOptions.readInputFromOverrideList().split(',');
+    const queries = CloudRunnerOptions.pullInputList;
-    CloudRunnerQueryOverride.queryOverrides = new Array();
+    CloudRunnerQueryOverride.queryOverrides = {};
     for (const element of queries) {
       if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
         CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);

src/model/cloud-runner/options/cloud-runner-step-parameters.ts

@@ -1,7 +1,7 @@
-import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
-import CloudRunnerSecret from './services/cloud-runner-secret';
+import CloudRunnerSecret from './cloud-runner-secret';
 
-export class CloudRunnerStepState {
+export class CloudRunnerStepParameters {
   public image: string;
   public environment: CloudRunnerEnvironmentVariable[];
   public secrets: CloudRunnerSecret[];

src/model/cloud-runner/providers/aws/aws-base-stack.ts

@@ -1,8 +1,8 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as core from '@actions/core';
 import * as SDK from 'aws-sdk';
 import { BaseStackFormation } from './cloud-formations/base-stack-formation';
-const crypto = require('crypto');
+import crypto from 'node:crypto';
 
 export class AWSBaseStack {
   constructor(baseStackName: string) {

src/model/cloud-runner/providers/aws/aws-cloud-formation-templates.ts

@@ -1,7 +1,7 @@
 import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
 
 export class AWSCloudFormationTemplates {
-  public static getParameterTemplate(p1) {
+  public static getParameterTemplate(p1: string) {
     return `
   ${p1}:
     Type: String
@@ -9,7 +9,7 @@ export class AWSCloudFormationTemplates {
 `;
   }
 
-  public static getSecretTemplate(p1) {
+  public static getSecretTemplate(p1: string) {
     return `
   ${p1}Secret:
     Type: AWS::SecretsManager::Secret
@@ -19,14 +19,14 @@ export class AWSCloudFormationTemplates {
 `;
   }
 
-  public static getSecretDefinitionTemplate(p1, p2) {
+  public static getSecretDefinitionTemplate(p1: string, p2: string) {
     return `
             - Name: '${p1}'
               ValueFrom: !Ref ${p2}Secret
 `;
   }
 
-  public static insertAtTemplate(template, insertionKey, insertion) {
+  public static insertAtTemplate(template: string, insertionKey: string, insertion: string) {
     const index = template.search(insertionKey) + insertionKey.length + '\n'.length;
     template = [template.slice(0, index), insertion, template.slice(index)].join('');
 

src/model/cloud-runner/providers/aws/aws-error.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as SDK from 'aws-sdk';
 import * as core from '@actions/core';
 import CloudRunner from '../../cloud-runner';

src/model/cloud-runner/providers/aws/aws-job-stack.ts

@@ -1,12 +1,12 @@
 import * as SDK from 'aws-sdk';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSError } from './aws-error';
 import CloudRunner from '../../cloud-runner';
 import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
 
 export class AWSJobStack {
@@ -27,21 +27,19 @@ export class AWSJobStack {
   ): Promise<CloudRunnerAWSTaskDef> {
     const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
     let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
-    const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024';
-    const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072';
     taskDefCloudFormation = taskDefCloudFormation.replace(
       `ContainerCpu:
     Default: 1024`,
       `ContainerCpu:
-    Default: ${Number.parseInt(cpu)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
     );
     taskDefCloudFormation = taskDefCloudFormation.replace(
       `ContainerMemory:
     Default: 2048`,
       `ContainerMemory:
-    Default: ${Number.parseInt(memory)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
     );
-    if (CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (!CloudRunnerOptions.asyncCloudRunner) {
       taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
         taskDefCloudFormation,
         '# template resources logstream',
@@ -116,7 +114,7 @@ export class AWSJobStack {
       ...secretsMappedToCloudFormationParameters,
     ];
     CloudRunnerLogger.log(
-      `Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`,
+      `Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
     );
     let previousStackExists = true;
     while (previousStackExists) {
@@ -140,11 +138,16 @@ export class AWSJobStack {
       Capabilities: ['CAPABILITY_IAM'],
       Parameters: parameters,
     };
-
     try {
       CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
       await CF.createStack(createStackInput).promise();
       await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
+      const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
+      for (const parameter of parameters) {
+        if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
+          throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
+        }
+      }
     } catch (error) {
       await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
       throw error;
@@ -180,7 +183,7 @@ export class AWSJobStack {
     if (CloudRunnerOptions.useCleanupCron) {
       try {
         CloudRunnerLogger.log(`Creating job cleanup formation`);
-        CF.createStack(createCleanupStackInput).promise();
+        await CF.createStack(createCleanupStackInput).promise();
 
         // await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
       } catch (error) {

src/model/cloud-runner/providers/aws/aws-task-runner.ts

@@ -1,14 +1,14 @@
 import * as AWS from 'aws-sdk';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import * as core from '@actions/core';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
-import * as zlib from 'zlib';
+import * as zlib from 'node:zlib';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { Input } from '../../..';
 import CloudRunner from '../../cloud-runner';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import GitHub from '../../../github';
 
 class AWSTaskRunner {
@@ -32,7 +32,7 @@ class AWSTaskRunner {
     const streamName =
       taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
 
-    const task = await AWSTaskRunner.ECS.runTask({
+    const runParameters = {
       cluster,
       taskDefinition,
       platformVersion: '1.4.0',
@@ -41,7 +41,7 @@ class AWSTaskRunner {
           {
             name: taskDef.taskDefStackName,
             environment,
-            command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
+            command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
           },
         ],
       },
@@ -53,16 +53,23 @@ class AWSTaskRunner {
           securityGroups: [ContainerSecurityGroup],
         },
       },
-    }).promise();
+    };
+
+    if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
+      CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
+      throw new Error(`Container Overrides length must be at most 8192`);
+    }
+
+    const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
     const taskArn = task.tasks?.[0].taskArn || '';
     CloudRunnerLogger.log('Cloud runner job is starting');
     await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
     CloudRunnerLogger.log(
-      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
+      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
-        CloudRunnerOptions.watchCloudRunnerToEnd
+        CloudRunnerOptions.asyncCloudRunner
-      } Async:${CloudRunnerOptions.asyncCloudRunner}`,
+      }`,
     );
-    if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (CloudRunnerOptions.asyncCloudRunner) {
       const shouldCleanup: boolean = false;
       const output: string = '';
       CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
@@ -72,26 +79,31 @@ class AWSTaskRunner {
 
     CloudRunnerLogger.log(`Streaming...`);
     const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
-    await new Promise((resolve) => resolve(5000));
+    let exitCode;
-    const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
+    let containerState;
-    const containerState = taskData.containers?.[0];
+    let taskData;
-    const exitCode = containerState?.exitCode || undefined;
+    while (exitCode === undefined) {
+      await new Promise((resolve) => resolve(10000));
+      taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
+      containerState = taskData.containers?.[0];
+      exitCode = containerState?.exitCode;
+    }
     CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
-    const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
+    if (exitCode === undefined) {
+      CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
+    }
+    const wasSuccessful = exitCode === 0;
     if (wasSuccessful) {
       CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
 
       return { output, shouldCleanup };
-    } else {
-      if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
-        throw new Error('Container exited with code 1');
-      }
-      const message = `Cloud runner job exit code ${exitCode}`;
-      taskData.overrides = undefined;
-      taskData.attachments = undefined;
-      CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
-      throw new Error(message);
     }
+
+    if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
+      throw new Error('Container exited with code 1');
+    }
+
+    throw new Error(`Task failed`);
   }
 
   private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
@@ -129,7 +141,7 @@ class AWSTaskRunner {
     const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
     let iterator = await AWSTaskRunner.getLogIterator(stream);
 
-    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
+    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
     CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
     await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch:  ${logBaseUrl}`, ``);
     let shouldReadLogs = true;
@@ -192,22 +204,19 @@ class AWSTaskRunner {
   }
 
   private static logRecords(
-    records,
+    records: AWS.Kinesis.GetRecordsOutput,
     iterator: string,
     shouldReadLogs: boolean,
     output: string,
     shouldCleanup: boolean,
   ) {
     if (records.Records.length > 0 && iterator) {
-      for (let index = 0; index < records.Records.length; index++) {
+      for (const record of records.Records) {
-        const json = JSON.parse(
+        const json = JSON.parse(zlib.gunzipSync(Buffer.from(record.Data as string, 'base64')).toString('utf8'));
-          zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
-        );
         if (json.messageType === 'DATA_MESSAGE') {
-          for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
+          for (const logEvent of json.logEvents) {
-            const message = json.logEvents[logEventsIndex].message;
             ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
-              message,
+              logEvent.message,
               shouldReadLogs,
               shouldCleanup,
               output,
@@ -226,7 +235,7 @@ class AWSTaskRunner {
     }).promise();
   }
 
-  private static async getLogIterator(stream) {
+  private static async getLogIterator(stream: AWS.Kinesis.DescribeStreamOutput) {
     return (
       (
         await AWSTaskRunner.Kinesis.getShardIterator({

src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts

@@ -30,7 +30,7 @@ Parameters:
     Type: Number
     Description: How much CPU to give the container. 1024 is 1 CPU
   ContainerMemory:
-    Default: 2048
+    Default: 4096
     Type: Number
     Description: How much memory in megabytes to give the container
   BUILDGUID:

src/model/cloud-runner/providers/aws/index.ts

@@ -1,11 +1,11 @@
 import * as SDK from 'aws-sdk';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
 import AwsTaskRunner from './aws-task-runner';
 import { ProviderInterface } from '../provider-interface';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSJobStack as AwsJobStack } from './aws-job-stack';
 import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
 import { Input } from '../../..';
@@ -13,13 +13,13 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { TaskService } from './services/task-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 
 class AWSBuildEnvironment implements ProviderInterface {
   private baseStackName: string;
 
   constructor(buildParameters: BuildParameters) {
-    this.baseStackName = buildParameters.awsBaseStackName;
+    this.baseStackName = buildParameters.awsStackName;
   }
   async listResources(): Promise<ProviderResource[]> {
     await TaskService.getCloudFormationJobStacks();
@@ -75,7 +75,11 @@ class AWSBuildEnvironment implements ProviderInterface {
     branchName: string,
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
-  ) {}
+  ) {
+    process.env.AWS_REGION = Input.region;
+    const CF = new SDK.CloudFormation();
+    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
+  }
 
   async runTaskInWorkflow(
     buildGuid: string,
@@ -94,8 +98,6 @@ class AWSBuildEnvironment implements ProviderInterface {
     CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
     const entrypoint = ['/bin/sh'];
     const startTimeMs = Date.now();
-
-    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
     const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
       CF,
       buildGuid,
@@ -143,6 +145,9 @@ class AWSBuildEnvironment implements ProviderInterface {
     await CF.waitFor('stackDeleteComplete', {
       StackName: taskDef.taskDefStackName,
     }).promise();
+    await CF.waitFor('stackDeleteComplete', {
+      StackName: `${taskDef.taskDefStackName}-cleanup`,
+    }).promise();
     CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
     CloudRunnerLogger.log('Cleanup complete');
   }

src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts

@@ -1,10 +1,10 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { TaskService } from './task-service';
 
 export class GarbageCollectionService {
-  static isOlderThan1day(date: any) {
+  static isOlderThan1day(date: Date) {
     const ageDate = new Date(date.getTime() - Date.now());
 
     return ageDate.getDay() > 0;
@@ -17,14 +17,16 @@ export class GarbageCollectionService {
     const cwl = new AWS.CloudWatchLogs();
     const taskDefinitionsInUse = new Array();
     const tasks = await TaskService.getTasks();
+
     for (const task of tasks) {
       const { taskElement, element } = task;
       taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
-      if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
+      if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
         CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
         await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
       }
     }
+
     const jobStacks = await TaskService.getCloudFormationJobStacks();
     for (const element of jobStacks) {
       if (
@@ -36,13 +38,15 @@ export class GarbageCollectionService {
 
         return;
       }
+
       if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(element.CreationTime))) {
         if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
           CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
 
           return;
         }
-        CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
+
+        CloudRunnerLogger.log(`Deleting ${element.StackName}`);
         const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
         await CF.deleteStack(deleteStackInput).promise();
       }
@@ -51,7 +55,7 @@ export class GarbageCollectionService {
     for (const element of logGroups) {
       if (
         deleteResources &&
-        (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
+        (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
       ) {
         CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
         await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();

src/model/cloud-runner/providers/aws/services/task-service.ts

@@ -1,10 +1,12 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
 import AwsTaskRunner from '../aws-task-runner';
 import { ListObjectsRequest } from 'aws-sdk/clients/s3';
 import CloudRunner from '../../../cloud-runner';
+import { StackSummaries } from 'aws-sdk/clients/cloudformation';
+import { LogGroups } from 'aws-sdk/clients/cloudwatchlogs';
 
 export class TaskService {
   static async watch() {
@@ -18,7 +20,7 @@ export class TaskService {
     return output;
   }
   public static async getCloudFormationJobStacks() {
-    const result: any[] = [];
+    const result: StackSummaries = [];
     CloudRunnerLogger.log(``);
     CloudRunnerLogger.log(`List Cloud Formation Stacks`);
     process.env.AWS_REGION = Input.region;
@@ -62,7 +64,7 @@ export class TaskService {
     return result;
   }
   public static async getTasks() {
-    const result: any[] = [];
+    const result: { taskElement: AWS.ECS.Task; element: string }[] = [];
     CloudRunnerLogger.log(``);
     CloudRunnerLogger.log(`List Tasks`);
     process.env.AWS_REGION = Input.region;
@@ -123,7 +125,7 @@ export class TaskService {
     return message;
   }
   public static async getLogGroups() {
-    const result: any[] = [];
+    const result: LogGroups = [];
     process.env.AWS_REGION = Input.region;
     const ecs = new AWS.CloudWatchLogs();
     let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
@@ -159,7 +161,7 @@ export class TaskService {
     process.env.AWS_REGION = Input.region;
     const s3 = new AWS.S3();
     const listRequest: ListObjectsRequest = {
-      Bucket: CloudRunner.buildParameters.awsBaseStackName,
+      Bucket: CloudRunner.buildParameters.awsStackName,
     };
     const results = await s3.listObjects(listRequest).promise();
 

src/model/cloud-runner/providers/docker/index.ts

@@ -1,19 +1,21 @@
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import Docker from '../../../docker';
 import { Action } from '../../..';
-import { writeFileSync } from 'fs';
+import { writeFileSync } from 'node:fs';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
-import { CloudRunnerSystem } from '../../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-import * as fs from 'fs';
+import * as fs from 'node:fs';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
+import { StringKeyValuePair } from '../../../shared-types';
 
 class LocalDockerCloudRunner implements ProviderInterface {
-  public buildParameters: BuildParameters | undefined;
+  public buildParameters!: BuildParameters;
 
   listResources(): Promise<ProviderResource[]> {
     return new Promise((resolve) => resolve([]));
@@ -50,14 +52,14 @@ class LocalDockerCloudRunner implements ProviderInterface {
     if (
       fs.existsSync(
         `${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       )
     ) {
       await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
       await CloudRunnerSystem.Run(
         `rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       );
     }
@@ -86,7 +88,7 @@ class LocalDockerCloudRunner implements ProviderInterface {
     CloudRunnerLogger.log(commands);
 
     const { workspace, actionFolder } = Action;
-    const content: any[] = [];
+    const content: StringKeyValuePair[] = [];
     for (const x of secrets) {
       content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
     }
@@ -117,7 +119,7 @@ set -e
 mkdir -p /github/workspace/cloud-runner-cache
 mkdir -p /data/cache
 cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
-${commands}
+${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
 cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
 `;
     writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
@@ -148,6 +150,7 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
         },
       },
       true,
+      false,
     );
 
     return myOutput;

src/model/cloud-runner/providers/k8s/index.ts

@@ -2,19 +2,18 @@ import * as k8s from '@kubernetes/client-node';
 import { BuildParameters } from '../../..';
 import * as core from '@actions/core';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import KubernetesStorage from './kubernetes-storage';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import KubernetesTaskRunner from './kubernetes-task-runner';
 import KubernetesSecret from './kubernetes-secret';
 import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
 import KubernetesServiceAccount from './kubernetes-service-account';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
-import KubernetesPods from './kubernetes-pods';
 
 class Kubernetes implements ProviderInterface {
   public static Instance: Kubernetes;
@@ -94,16 +93,8 @@ class Kubernetes implements ProviderInterface {
   ) {
     try {
       this.buildParameters = buildParameters;
-      const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
+      this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
-      this.pvcName = `unity-builder-pvc-${id}`;
-      this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
       this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
-      await KubernetesStorage.createPersistentVolumeClaim(
-        buildParameters,
-        this.pvcName,
-        this.kubeClient,
-        this.namespace,
-      );
 
       await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
     } catch (error) {
@@ -124,74 +115,99 @@ class Kubernetes implements ProviderInterface {
       CloudRunnerLogger.log('Cloud Runner K8s workflow!');
 
       // Setup
+      const id = BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
+        ? CloudRunner.lockedWorkspace
+        : this.buildParameters.buildGuid;
+      this.pvcName = `unity-builder-pvc-${id}`;
+      await KubernetesStorage.createPersistentVolumeClaim(
+        this.buildParameters,
+        this.pvcName,
+        this.kubeClient,
+        this.namespace,
+      );
       this.buildGuid = buildGuid;
       this.secretName = `build-credentials-${this.buildGuid}`;
       this.jobName = `unity-builder-job-${this.buildGuid}`;
       this.containerName = `main`;
       await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
-      await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
-      this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
-      CloudRunnerLogger.log('Watching pod until running');
-      await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
       let output = '';
-      // eslint-disable-next-line no-constant-condition
+      try {
-      while (true) {
+        CloudRunnerLogger.log('Job does not exist');
-        try {
+        await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
-          CloudRunnerLogger.log('Pod running, streaming logs');
+        CloudRunnerLogger.log('Watching pod until running');
-          output = await KubernetesTaskRunner.runTask(
+        await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
-            this.kubeConfig,
-            this.kubeClient,
-            this.jobName,
-            this.podName,
-            'main',
-            this.namespace,
-          );
-          const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
 
-          if (!running) {
+        CloudRunnerLogger.log('Pod running, streaming logs');
-            CloudRunnerLogger.log(`Pod not found, assumed ended!`);
+        CloudRunnerLogger.log(
-            break;
+          `Starting logs follow for pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace} pvc: ${this.pvcName} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
-          } else {
+        );
-            CloudRunnerLogger.log('Pod still running, recovering stream...');
+        output += await KubernetesTaskRunner.runTask(
-          }
+          this.kubeConfig,
-          await this.cleanupTaskResources();
+          this.kubeClient,
-        } catch (error: any) {
+          this.jobName,
-          let errorParsed;
+          this.podName,
-          try {
+          this.containerName,
-            errorParsed = JSON.parse(error);
+          this.namespace,
-          } catch {
+        );
-            errorParsed = error;
+      } catch (error: any) {
-          }
+        CloudRunnerLogger.log(`error running k8s workflow ${error}`);
-
+        await new Promise((resolve) => setTimeout(resolve, 3000));
-          const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
+        CloudRunnerLogger.log(
-          const errorMessage = errorParsed.message || reason;
+          JSON.stringify(
-
+            (await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
-          const continueStreaming =
+              .map((x) => {
-            errorMessage.includes(`dial timeout, backstop`) ||
+                return {
-            errorMessage.includes(`HttpError: HTTP request failed`) ||
+                  message: x.message || ``,
-            errorMessage.includes(`an error occurred when try to find container`) ||
+                  name: x.metadata.name || ``,
-            errorMessage.includes(`not found`) ||
+                  reason: x.reason || ``,
-            errorMessage.includes(`Not Found`);
+                };
-          if (continueStreaming) {
+              })
-            CloudRunnerLogger.log('Log Stream Container Not Found');
+              .filter((x) => x.name.includes(this.podName)),
-            await new Promise((resolve) => resolve(5000));
+            undefined,
-            continue;
+            4,
-          } else {
+          ),
-            CloudRunnerLogger.log(`error running k8s workflow ${error}`);
+        );
-            throw error;
+        await this.cleanupTaskResources();
-          }
+        throw error;
-        }
       }
 
+      await this.cleanupTaskResources();
+
       return output;
     } catch (error) {
       CloudRunnerLogger.log('Running job failed');
       core.error(JSON.stringify(error, undefined, 4));
-      await this.cleanupTaskResources();
+
+      // await this.cleanupTaskResources();
       throw error;
     }
   }
 
+  private async createJob(
+    commands: string,
+    image: string,
+    mountdir: string,
+    workingdir: string,
+    environment: CloudRunnerEnvironmentVariable[],
+    secrets: CloudRunnerSecret[],
+  ) {
+    await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
+    const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
+    this.setPodNameAndContainerName(find);
+  }
+
+  private async doesJobExist(name: string) {
+    const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
+
+    return jobs.body.items.some((x) => x.metadata?.name === name);
+  }
+
+  private async doesFailedJobExist() {
+    const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
+
+    return podStatus.body.status?.phase === `Failed`;
+  }
+
   private async createNamespacedJob(
     commands: string,
     image: string,
@@ -215,14 +231,15 @@ class Kubernetes implements ProviderInterface {
           this.pvcName,
           this.jobName,
           k8s,
+          this.containerName,
         );
         await new Promise((promise) => setTimeout(promise, 15000));
-        await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
+        const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
         CloudRunnerLogger.log(`Build job created`);
         await new Promise((promise) => setTimeout(promise, 5000));
         CloudRunnerLogger.log('Job created');
 
-        return;
+        return result.body.metadata?.name;
       } catch (error) {
         CloudRunnerLogger.log(`Error occured creating job: ${error}`);
         throw error;
@@ -232,7 +249,7 @@ class Kubernetes implements ProviderInterface {
 
   setPodNameAndContainerName(pod: k8s.V1Pod) {
     this.podName = pod.metadata?.name || '';
-    this.containerName = pod.status?.containerStatuses?.[0].name || '';
+    this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
   }
 
   async cleanupTaskResources() {
@@ -265,7 +282,7 @@ class Kubernetes implements ProviderInterface {
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
   ) {
-    if (buildParameters.retainWorkspace) {
+    if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
       return;
     }
     CloudRunnerLogger.log(`deleting PVC`);

src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts

@@ -1,8 +1,8 @@
 import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import CloudRunner from '../../cloud-runner';
 
 class KubernetesJobSpecFactory {
@@ -15,67 +15,12 @@ class KubernetesJobSpecFactory {
     secrets: CloudRunnerSecret[],
     buildGuid: string,
     buildParameters: BuildParameters,
-    secretName,
+    secretName: string,
-    pvcName,
+    pvcName: string,
-    jobName,
+    jobName: string,
-    k8s,
+    k8s: any,
+    containerName: string,
   ) {
-    environment.push(
-      ...[
-        {
-          name: 'GITHUB_SHA',
-          value: buildGuid,
-        },
-        {
-          name: 'GITHUB_WORKSPACE',
-          value: '/data/repo',
-        },
-        {
-          name: 'PROJECT_PATH',
-          value: buildParameters.projectPath,
-        },
-        {
-          name: 'BUILD_PATH',
-          value: buildParameters.buildPath,
-        },
-        {
-          name: 'BUILD_FILE',
-          value: buildParameters.buildFile,
-        },
-        {
-          name: 'BUILD_NAME',
-          value: buildParameters.buildName,
-        },
-        {
-          name: 'BUILD_METHOD',
-          value: buildParameters.buildMethod,
-        },
-        {
-          name: 'CUSTOM_PARAMETERS',
-          value: buildParameters.customParameters,
-        },
-        {
-          name: 'CHOWN_FILES_TO',
-          value: buildParameters.chownFilesTo,
-        },
-        {
-          name: 'BUILD_TARGET',
-          value: buildParameters.targetPlatform,
-        },
-        {
-          name: 'ANDROID_VERSION_CODE',
-          value: buildParameters.androidVersionCode.toString(),
-        },
-        {
-          name: 'ANDROID_KEYSTORE_NAME',
-          value: buildParameters.androidKeystoreName,
-        },
-        {
-          name: 'ANDROID_KEYALIAS_NAME',
-          value: buildParameters.androidKeyaliasName,
-        },
-      ],
-    );
     const job = new k8s.V1Job();
     job.apiVersion = 'batch/v1';
     job.kind = 'Job';
@@ -87,6 +32,7 @@ class KubernetesJobSpecFactory {
       },
     };
     job.spec = {
+      ttlSecondsAfterFinished: 9999,
       backoffLimit: 0,
       template: {
         spec: {
@@ -100,16 +46,20 @@ class KubernetesJobSpecFactory {
           ],
           containers: [
             {
-              name: 'main',
+              ttlSecondsAfterFinished: 9999,
+              name: containerName,
               image,
               command: ['/bin/sh'],
-              args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
+              args: [
+                '-c',
+                `${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
+              ],
 
               workingDir: `${workingDirectory}`,
               resources: {
                 requests: {
-                  memory: buildParameters.cloudRunnerMemory || '750M',
+                  memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
-                  cpu: buildParameters.cloudRunnerCpu || '1',
+                  cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
                 },
               },
               env: [
@@ -135,7 +85,7 @@ class KubernetesJobSpecFactory {
               volumeMounts: [
                 {
                   name: 'build-mount',
-                  mountPath: `/${mountdir}`,
+                  mountPath: `${mountdir}`,
                 },
               ],
               lifecycle: {
@@ -158,7 +108,7 @@ class KubernetesJobSpecFactory {
       },
     };
 
-    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
+    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
 
     return job;
   }

src/model/cloud-runner/providers/k8s/kubernetes-pods.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 class KubernetesPods {
   public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
@@ -12,6 +12,12 @@ class KubernetesPods {
 
     return running;
   }
+  public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
+    const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
+    const phase = pods?.status?.phase || 'undefined status';
+
+    return phase;
+  }
 }
 
 export default KubernetesPods;

src/model/cloud-runner/providers/k8s/kubernetes-secret.ts

@@ -1,8 +1,8 @@
 import { CoreV1Api } from '@kubernetes/client-node';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import * as k8s from '@kubernetes/client-node';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-const base64 = require('base-64');
+import * as base64 from 'base-64';
 
 class KubernetesSecret {
   static async createSecret(

src/model/cloud-runner/providers/k8s/kubernetes-storage.ts

@@ -1,9 +1,9 @@
-import waitUntil from 'async-wait-until';
+import { waitUntil } from 'async-wait-until';
 import * as core from '@actions/core';
 import * as k8s from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-import { IncomingMessage } from 'http';
+import { IncomingMessage } from 'node:http';
 import GitHub from '../../../github';
 
 class KubernetesStorage {

src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts

@@ -1,12 +1,16 @@
-import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
+import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
-import { Writable } from 'stream';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import { waitUntil } from 'async-wait-until';
-import * as core from '@actions/core';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-import { CloudRunnerStatics } from '../../cloud-runner-statics';
+import CloudRunner from '../../cloud-runner';
-import waitUntil from 'async-wait-until';
+import KubernetesPods from './kubernetes-pods';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
 
 class KubernetesTaskRunner {
+  static lastReceivedTimestamp: number = 0;
+  static readonly maxRetry: number = 3;
+  static lastReceivedMessage: string = ``;
+
   static async runTask(
     kubeConfig: KubeConfig,
     kubeClient: CoreV1Api,
@@ -15,84 +19,120 @@ class KubernetesTaskRunner {
     containerName: string,
     namespace: string,
   ) {
-    CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
-    const stream = new Writable();
     let output = '';
-    let didStreamAnyLogs: boolean = false;
     let shouldReadLogs = true;
     let shouldCleanup = true;
-    stream._write = (chunk, encoding, next) => {
+    let sinceTime = ``;
-      didStreamAnyLogs = true;
+    let retriesAfterFinish = 0;
-      let message = chunk.toString().trimRight(`\n`);
+    // eslint-disable-next-line no-constant-condition
-      message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
+    while (true) {
-      ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+      await new Promise((resolve) => setTimeout(resolve, 3000));
-        message,
+      const lastReceivedMessage =
-        shouldReadLogs,
+        KubernetesTaskRunner.lastReceivedTimestamp > 0
-        shouldCleanup,
+          ? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
-        output,
+          : ``;
-      ));
+      CloudRunnerLogger.log(
-      next();
+        `Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
-    };
-    const logOptions = {
-      follow: true,
-      pretty: false,
-      previous: false,
-    };
-    try {
-      const resultError = await new Promise((resolve) =>
-        new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
       );
-      stream.destroy();
+      if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
-      if (resultError) {
+        const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
-        throw resultError;
+        const dateTimeIsoString = currentDate.toISOString();
+        sinceTime = ` --since-time="${dateTimeIsoString}"`;
       }
-      if (!didStreamAnyLogs) {
+      let extraFlags = ``;
-        core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
+      extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
-        core.error(
+        ? ` -f -c ${containerName}`
-          JSON.stringify(
+        : ` --previous`;
-            {
+      let lastMessageSeenIncludedInChunk = false;
-              events: (await kubeClient.listNamespacedEvent(namespace)).body.items
+      let lastMessageSeen = false;
-                .filter((x) => {
+
-                  return x.involvedObject.name === podName || x.involvedObject.name === jobName;
+      let logs;
-                })
+
-                .map((x) => {
+      try {
-                  return {
+        logs = await CloudRunnerSystem.Run(
-                    type: x.involvedObject.kind,
+          `kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
-                    name: x.involvedObject.name,
+          false,
-                    message: x.message,
+          true,
-                  };
-                }),
-            },
-            undefined,
-            4,
-          ),
         );
-        throw new Error(`No logs streamed from k8s`);
+      } catch (error: any) {
+        await new Promise((resolve) => setTimeout(resolve, 3000));
+        const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
+        CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
+        if (continueStreaming) {
+          continue;
+        }
+        if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
+          retriesAfterFinish++;
+
+          continue;
+        }
+        throw error;
       }
-    } catch (error) {
+      const splitLogs = logs.split(`\n`);
-      if (stream) {
+      for (const chunk of splitLogs) {
-        stream.destroy();
+        if (
+          chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
+          KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
+        ) {
+          CloudRunnerLogger.log(`Previous log message found ${chunk}`);
+          lastMessageSeenIncludedInChunk = true;
+        }
+      }
+      for (const chunk of splitLogs) {
+        const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
+        if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
+          lastMessageSeen = true;
+        }
+        if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
+          continue;
+        }
+        const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
+        KubernetesTaskRunner.lastReceivedMessage = chunk;
+        KubernetesTaskRunner.lastReceivedTimestamp = newDate;
+        ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+          message,
+          shouldReadLogs,
+          shouldCleanup,
+          output,
+        ));
+      }
+      if (FollowLogStreamService.DidReceiveEndOfTransmission) {
+        CloudRunnerLogger.log('end of log stream');
+        break;
       }
-      throw error;
     }
-    CloudRunnerLogger.log('end of log stream');
 
     return output;
   }
 
   static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
     let success: boolean = false;
+    let message = ``;
     CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
     await waitUntil(
       async () => {
         const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
         const phase = status?.body.status?.phase;
         success = phase === 'Running';
-        CloudRunnerLogger.log(
+        message = `Phase:${status.body.status?.phase} \n Reason:${
-          `${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
+          status.body.status?.conditions?.[0].reason || ''
-            status.body.status?.conditions?.[0].message || ''
+        } \n Message:${status.body.status?.conditions?.[0].message || ''}`;
-          }`,
+
-        );
+        // CloudRunnerLogger.log(
+        //   JSON.stringify(
+        //     (await kubeClient.listNamespacedEvent(namespace)).body.items
+        //       .map((x) => {
+        //         return {
+        //           message: x.message || ``,
+        //           name: x.metadata.name || ``,
+        //           reason: x.reason || ``,
+        //         };
+        //       })
+        //       .filter((x) => x.name.includes(podName)),
+        //     undefined,
+        //     4,
+        //   ),
+        // );
         if (success || phase !== 'Pending') return true;
 
         return false;
@@ -102,6 +142,9 @@ class KubernetesTaskRunner {
         intervalBetweenAttempts: 15000,
       },
     );
+    if (!success) {
+      CloudRunnerLogger.log(message);
+    }
 
     return success;
   }

src/model/cloud-runner/providers/local/index.ts

@@ -1,9 +1,9 @@
 import BuildParameters from '../../../build-parameters';
-import { CloudRunnerSystem } from '../../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 

src/model/cloud-runner/providers/provider-interface.ts

@@ -1,6 +1,6 @@
 import BuildParameters from '../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
 import { ProviderResource } from './provider-resource';
 import { ProviderWorkflow } from './provider-workflow';
 
@@ -14,7 +14,7 @@ export interface ProviderInterface {
     branchName: string,
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
-  );
+  ): any;
   setupWorkflow(
     // eslint-disable-next-line no-unused-vars
     buildGuid: string,
@@ -24,7 +24,7 @@ export interface ProviderInterface {
     branchName: string,
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
-  );
+  ): any;
   runTaskInWorkflow(
     // eslint-disable-next-line no-unused-vars
     buildGuid: string,

src/model/cloud-runner/providers/test/index.ts

@@ -1,8 +1,8 @@
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 

src/model/cloud-runner/remote-client/caching.ts

@@ -1,16 +1,16 @@
-import { assert } from 'console';
+import { assert } from 'node:console';
-import fs from 'fs';
+import fs from 'node:fs';
-import path from 'path';
+import path from 'node:path';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
-import { LfsHashing } from '../services/lfs-hashing';
+import { LfsHashing } from '../services/utility/lfs-hashing';
 import { RemoteClientLogger } from './remote-client-logger';
 import { Cli } from '../../cli/cli';
 import { CliFunction } from '../../cli/cli-functions-repository';
 // eslint-disable-next-line github/no-then
-const fileExists = async (fpath) => !!(await fs.promises.stat(fpath).catch(() => false));
+const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
 
 export class Caching {
   @CliFunction(`cache-push`, `push to cache`)
@@ -19,9 +19,9 @@ export class Caching {
       const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
       CloudRunner.buildParameters = buildParameter;
       await Caching.PushToCache(
-        Cli.options['cachePushTo'],
+        Cli.options!['cachePushTo'],
-        Cli.options['cachePushFrom'],
+        Cli.options!['cachePushFrom'],
-        Cli.options['artifactName'] || '',
+        Cli.options!['artifactName'] || '',
       );
     } catch (error: any) {
       CloudRunnerLogger.log(`${error}`);
@@ -34,9 +34,9 @@ export class Caching {
       const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
       CloudRunner.buildParameters = buildParameter;
       await Caching.PullFromCache(
-        Cli.options['cachePushFrom'],
+        Cli.options!['cachePushFrom'],
-        Cli.options['cachePushTo'],
+        Cli.options!['cachePushTo'],
-        Cli.options['artifactName'] || '',
+        Cli.options!['artifactName'] || '',
       );
     } catch (error: any) {
       CloudRunnerLogger.log(`${error}`);
@@ -44,20 +44,21 @@ export class Caching {
   }
 
   public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
+    CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
     cacheArtifactName = cacheArtifactName.replace(' ', '');
     const startPath = process.cwd();
     let compressionSuffix = '';
-    if (CloudRunner.buildParameters.useLz4Compression === true) {
+    if (CloudRunner.buildParameters.useCompressionStrategy === true) {
       compressionSuffix = `.lz4`;
     }
-    CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useLz4Compression} ${compressionSuffix}`);
+    CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
     try {
       if (!(await fileExists(cacheFolder))) {
         await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
       }
       process.chdir(path.resolve(sourceFolder, '..'));
 
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
         CloudRunnerLogger.log(
           `Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
             sourceFolder,
@@ -69,11 +70,6 @@ export class Caching {
         `There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
       );
 
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
-        // await CloudRunnerSystem.Run(`tree -L 2 ./..`);
-        // await CloudRunnerSystem.Run(`tree -L 2`);
-      }
-
       if (contents.length === 0) {
         CloudRunnerLogger.log(
           `Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
@@ -102,9 +98,15 @@ export class Caching {
     process.chdir(`${startPath}`);
   }
   public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
+    CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
+    if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
+      CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
+
+      return;
+    }
     cacheArtifactName = cacheArtifactName.replace(' ', '');
     let compressionSuffix = '';
-    if (CloudRunner.buildParameters.useLz4Compression === true) {
+    if (CloudRunner.buildParameters.useCompressionStrategy === true) {
       compressionSuffix = `.lz4`;
     }
     const startPath = process.cwd();
@@ -160,7 +162,6 @@ export class Caching {
           RemoteClientLogger.logWarning(
             `cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
           );
-          await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
           throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
         }
       }

src/model/cloud-runner/remote-client/index.ts

@@ -1,56 +1,80 @@
-import fs from 'fs';
+import fs from 'node:fs';
 import CloudRunner from '../cloud-runner';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
 import { Caching } from './caching';
-import { LfsHashing } from '../services/lfs-hashing';
+import { LfsHashing } from '../services/utility/lfs-hashing';
 import { RemoteClientLogger } from './remote-client-logger';
-import path from 'path';
+import path from 'node:path';
-import { assert } from 'console';
+import { assert } from 'node:console';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 import { CliFunction } from '../../cli/cli-functions-repository';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import YAML from 'yaml';
+import GitHub from '../../github';
+import BuildParameters from '../../build-parameters';
 
 export class RemoteClient {
-  public static async bootstrapRepository() {
+  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-    try {
+  static async runRemoteClientJob() {
-      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
+    CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
-      await CloudRunnerSystem.Run(
+    if (!(await RemoteClient.handleRetainedWorkspace())) {
-        `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
+      await RemoteClient.bootstrapRepository();
-      );
-      process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
-      await RemoteClient.cloneRepoWithoutLFSFiles();
-      RemoteClient.replaceLargePackageReferencesWithSharedReferences();
-      await RemoteClient.sizeOfFolder(
-        'repo before lfs cache pull',
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
-      );
-      const lfsHashes = await LfsHashing.createLFSHashFiles();
-      if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
-        RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
-      }
-      await Caching.PullFromCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
-        `${lfsHashes.lfsGuidSum}`,
-      );
-      await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
-      await RemoteClient.pullLatestLFS();
-      await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
-      await Caching.PushToCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
-        `${lfsHashes.lfsGuidSum}`,
-      );
-      await Caching.PullFromCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
-      );
-      await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
-      await Caching.handleCachePurging();
-    } catch (error) {
-      throw error;
     }
+    await RemoteClient.runCustomHookFiles(`before-build`);
+  }
+  static async runCustomHookFiles(hookLifecycle: string) {
+    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
+    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
+    try {
+      const files = fs.readdirSync(gameCiCustomHooksPath);
+      for (const file of files) {
+        const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
+        const fileContentsObject = YAML.parse(fileContents.toString());
+        if (fileContentsObject.hook === hookLifecycle) {
+          RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
+          await CloudRunnerSystem.Run(fileContentsObject.commands);
+        }
+      }
+    } catch (error) {
+      RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
+    }
+  }
+  public static async bootstrapRepository() {
+    await CloudRunnerSystem.Run(
+      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
+    );
+    await CloudRunnerSystem.Run(
+      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
+    );
+    await RemoteClient.cloneRepoWithoutLFSFiles();
+    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
+    await RemoteClient.sizeOfFolder(
+      'repo before lfs cache pull',
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
+    );
+    const lfsHashes = await LfsHashing.createLFSHashFiles();
+    if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
+      RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
+    }
+    await Caching.PullFromCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
+      `${lfsHashes.lfsGuidSum}`,
+    );
+    await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
+    await RemoteClient.pullLatestLFS();
+    await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
+      `${lfsHashes.lfsGuidSum}`,
+    );
+    await Caching.PullFromCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
+    );
+    await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
+    await Caching.handleCachePurging();
   }
 
   private static async sizeOfFolder(message: string, folder: string) {
@@ -62,58 +86,68 @@ export class RemoteClient {
 
   private static async cloneRepoWithoutLFSFiles() {
     process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
+    if (
+      fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
+      !fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
+    ) {
+      await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
+      CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
+    }
 
     if (
-      CloudRunner.buildParameters.retainWorkspace &&
+      BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
       fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
     ) {
       process.chdir(CloudRunnerFolders.repoPathAbsolute);
       RemoteClientLogger.log(
-        `${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
+        `${
+          CloudRunnerFolders.repoPathAbsolute
+        } repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
+          CloudRunner.buildParameters,
+        )}`,
       );
       await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
 
       return;
     }
 
-    if (fs.existsSync(CloudRunnerFolders.repoPathAbsolute)) {
+    RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
-      RemoteClientLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists cleaning up`);
+    await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
-      await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
+    RemoteClientLogger.log(`Cloning the repository being built:`);
-    }
+    await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
-
+    await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
     try {
-      RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
-      await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
-      RemoteClientLogger.log(`Cloning the repository being built:`);
-      await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
-      await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
       await CloudRunnerSystem.Run(
-        `git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
+        `git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
       );
-      process.chdir(CloudRunnerFolders.repoPathAbsolute);
+    } catch (error: any) {
-      await CloudRunnerSystem.Run(`git lfs install`);
-      assert(fs.existsSync(`.git`), 'git folder exists');
-      RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
-      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
-      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
-      assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
-      RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
-    } catch (error) {
-      await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
       throw error;
     }
+    process.chdir(CloudRunnerFolders.repoPathAbsolute);
+    await CloudRunnerSystem.Run(`git lfs install`);
+    assert(fs.existsSync(`.git`), 'git folder exists');
+    RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
+    if (CloudRunner.buildParameters.gitSha !== undefined) {
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
+    } else {
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
+      RemoteClientLogger.log(`buildParameter Git Sha is empty`);
+    }
+
+    assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
+    RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
   }
 
-  static replaceLargePackageReferencesWithSharedReferences() {
+  static async replaceLargePackageReferencesWithSharedReferences() {
-    if (CloudRunner.buildParameters.useSharedLargePackages) {
+    CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
+    GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
+    if (CloudRunner.buildParameters.useLargePackages) {
       const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
       let manifest = fs.readFileSync(filePath, 'utf8');
       manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
       fs.writeFileSync(filePath, manifest);
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
-        CloudRunnerLogger.log(`Package Manifest`);
+      GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
-        CloudRunnerLogger.log(manifest);
-      }
     }
   }
 
@@ -121,41 +155,31 @@ export class RemoteClient {
     process.chdir(CloudRunnerFolders.repoPathAbsolute);
     await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
     await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
-    await CloudRunnerSystem.Run(`git lfs pull`);
+    if (!CloudRunner.buildParameters.skipLfs) {
-    RemoteClientLogger.log(`pulled latest LFS files`);
+      await CloudRunnerSystem.Run(`git lfs pull`);
-    assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
+      RemoteClientLogger.log(`pulled latest LFS files`);
-  }
+      assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
-
-  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-  static async runRemoteClientJob() {
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    RemoteClient.handleRetainedWorkspace();
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    await RemoteClient.bootstrapRepository();
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    await RemoteClient.runCustomHookFiles(`before-build`);
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-  }
-  static async runCustomHookFiles(hookLifecycle: string) {
-    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
-    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
-    const files = fs.readdirSync(gameCiCustomHooksPath);
-    for (const file of files) {
-      const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
-      const fileContentsObject = YAML.parse(fileContents.toString());
-      if (fileContentsObject.hook === hookLifecycle) {
-        RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
-        await CloudRunnerSystem.Run(fileContentsObject.commands);
-      }
     }
   }
-  static handleRetainedWorkspace() {
+  static async handleRetainedWorkspace() {
-    if (!CloudRunner.buildParameters.retainWorkspace) {
+    RemoteClientLogger.log(
-      return;
+      `Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
+    );
+    if (
+      BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
+      fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
+      fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
+    ) {
+      CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
+      process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
+      await CloudRunnerSystem.Run(`git fetch`);
+      await CloudRunnerSystem.Run(`git lfs pull`);
+      await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
+
+      return true;
     }
-    RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
+
+    return false;
   }
 }

src/model/cloud-runner/remote-client/remote-client-logger.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 
 export class RemoteClientLogger {
   public static log(message: string) {
@@ -13,7 +13,7 @@ export class RemoteClientLogger {
     CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
   }
 
-  public static logWarning(message) {
+  public static logWarning(message: string) {
     CloudRunnerLogger.logWarning(message);
   }
 }

src/model/cloud-runner/services/cloud-runner-custom-hooks.ts

@@ -1,118 +0,0 @@
-import { BuildParameters, Input } from '../..';
-import YAML from 'yaml';
-import CloudRunnerSecret from './cloud-runner-secret';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
-import path from 'path';
-import CloudRunnerOptions from '../cloud-runner-options';
-import * as fs from 'fs';
-
-// import CloudRunnerLogger from './cloud-runner-logger';
-
-export class CloudRunnerCustomHooks {
-  // TODO also accept hooks as yaml files in the repo
-  public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
-    const hooks = CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
-
-    return `echo "---"
-      echo "start cloud runner init"
-      ${CloudRunnerOptions.cloudRunnerDebugEnv ? `printenv` : `#`}
-      echo "start of cloud runner job"
-      ${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
-      ${commands}
-      ${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
-      echo "end of cloud runner job"
-      echo "---${buildParameters.logId}"`;
-  }
-
-  public static getHooks(customJobHooks): Hook[] {
-    const experimentHooks = customJobHooks;
-    let output = new Array<Hook>();
-    if (experimentHooks && experimentHooks !== '') {
-      try {
-        output = YAML.parse(experimentHooks);
-      } catch (error) {
-        throw error;
-      }
-    }
-
-    return output.filter((x) => x.step !== undefined && x.hook !== undefined && x.hook.length > 0);
-  }
-
-  static GetCustomHooksFromFiles(hookLifecycle: string): Hook[] {
-    const results: Hook[] = [];
-    RemoteClientLogger.log(`GetCustomStepFiles: ${hookLifecycle}`);
-    try {
-      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `hooks`);
-      const files = fs.readdirSync(gameCiCustomStepsPath);
-      for (const file of files) {
-        if (!CloudRunnerOptions.customHookFiles.includes(file.replace(`.yaml`, ``))) {
-          continue;
-        }
-        const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
-        const fileContentsObject = CloudRunnerCustomHooks.ParseHooks(fileContents)[0];
-        if (fileContentsObject.hook.includes(hookLifecycle)) {
-          results.push(fileContentsObject);
-        }
-      }
-    } catch (error) {
-      RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
-    }
-    RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
-
-    return results;
-  }
-
-  private static ConvertYamlSecrets(object) {
-    if (object.secrets === undefined) {
-      object.secrets = [];
-
-      return;
-    }
-    object.secrets = object.secrets.map((x) => {
-      return {
-        ParameterKey: x.name,
-        EnvironmentVariable: Input.ToEnvVarFormat(x.name),
-        ParameterValue: x.value,
-      };
-    });
-  }
-
-  public static ParseHooks(steps: string): Hook[] {
-    if (steps === '') {
-      return [];
-    }
-
-    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
-
-    // CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
-
-    // }
-    const isArray = steps.replace(/\s/g, ``)[0] === `-`;
-    const object: Hook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
-    for (const hook of object) {
-      CloudRunnerCustomHooks.ConvertYamlSecrets(hook);
-      if (hook.secrets === undefined) {
-        hook.secrets = [];
-      }
-    }
-    if (object === undefined) {
-      throw new Error(`Failed to parse ${steps}`);
-    }
-
-    return object;
-  }
-
-  public static getSecrets(hooks) {
-    const secrets = hooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
-
-    // eslint-disable-next-line unicorn/no-array-reduce
-    return secrets.length > 0 ? secrets.reduce((x, y) => [...x, ...y]) : [];
-  }
-}
-export class Hook {
-  public commands;
-  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
-  public name;
-  public hook!: string[];
-  public step!: string[];
-}

src/model/cloud-runner/services/core/cloud-runner-system.ts

@@ -1,5 +1,5 @@
 import { exec } from 'child_process';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
 
 export class CloudRunnerSystem {
   public static async RunAndReadLines(command: string): Promise<string[]> {

src/model/cloud-runner/services/core/follow-log-stream-service.ts

@@ -0,0 +1,57 @@
+import GitHub from '../../../github';
+import CloudRunner from '../../cloud-runner';
+import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
+import CloudRunnerLogger from './cloud-runner-logger';
+import * as core from '@actions/core';
+
+export class FollowLogStreamService {
+  static Reset() {
+    FollowLogStreamService.DidReceiveEndOfTransmission = false;
+  }
+  static errors = ``;
+  public static DidReceiveEndOfTransmission = false;
+  public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
+    if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
+      CloudRunnerLogger.log('End of log transmission received');
+      FollowLogStreamService.DidReceiveEndOfTransmission = true;
+      shouldReadLogs = false;
+    } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
+      GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
+      core.warning('LIBRARY NOT FOUND!');
+      core.setOutput('library-found', 'false');
+    } else if (message.includes('Build succeeded')) {
+      GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
+      core.setOutput('build-result', 'success');
+    } else if (message.includes('Build fail')) {
+      GitHub.updateGitHubCheck(
+        `Build failed\n${FollowLogStreamService.errors}`,
+        `Build failed`,
+        `failure`,
+        `completed`,
+      );
+      core.setOutput('build-result', 'failed');
+      core.setFailed('unity build failed');
+      core.error('BUILD FAILED!');
+    } else if (message.toLowerCase().includes('error ')) {
+      core.error(message);
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('error: ')) {
+      core.error(message);
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('command failed: ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('invalid ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('incompatible  ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('cannot be found')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    }
+    if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      output += `${message}\n`;
+    }
+    CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
+
+    return { shouldReadLogs, shouldCleanup, output };
+  }
+}

src/model/cloud-runner/services/core/shared-workspace-locking.ts

@@ -1,18 +1,17 @@
 import { CloudRunnerSystem } from './cloud-runner-system';
-import * as fs from 'fs';
+import fs from 'node:fs';
 import CloudRunnerLogger from './cloud-runner-logger';
-import CloudRunnerOptions from '../cloud-runner-options';
+import BuildParameters from '../../../build-parameters';
-import BuildParameters from '../../build-parameters';
+import CloudRunner from '../../cloud-runner';
-import CloudRunner from '../cloud-runner';
 export class SharedWorkspaceLocking {
-  private static get workspaceBucketRoot() {
+  public static get workspaceBucketRoot() {
-    return `s3://${CloudRunner.buildParameters.awsBaseStackName}/`;
+    return `s3://${CloudRunner.buildParameters.awsStackName}/`;
   }
-  private static get workspaceRoot() {
+  public static get workspaceRoot() {
     return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
   }
   public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
-    if (!(await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext))) {
+    if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
       return [];
     }
 
@@ -20,79 +19,95 @@ export class SharedWorkspaceLocking {
       await SharedWorkspaceLocking.ReadLines(
         `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
-    ).map((x) => x.replace(`/`, ``));
+    )
-  }
-  public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
-    await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`);
-
-    return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`))
       .map((x) => x.replace(`/`, ``))
-      .includes(buildParametersContext.cacheKey);
+      .filter((x) => x.endsWith(`_workspace`))
+      .map((x) => x.split(`_`)[1]);
   }
-  public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
+  public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
+    try {
+      const rootLines = await SharedWorkspaceLocking.ReadLines(
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
+      );
+      const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
+
+      if (lockFolderExists) {
+        const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
+
+        return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
+      } else {
+        return false;
+      }
+    } catch {
+      return false;
+    }
+  }
+
+  public static NewWorkspaceName() {
+    return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
+  }
+  public static async GetAllLocksForWorkspace(
+    workspace: string,
+    buildParametersContext: BuildParameters,
+  ): Promise<string[]> {
     if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
       return [];
     }
 
     return (
       await SharedWorkspaceLocking.ReadLines(
-        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
     )
       .map((x) => x.replace(`/`, ``))
-      .filter((x) => x.includes(`_lock`));
+      .filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
   }
-  public static async GetOrCreateLockedWorkspace(
+  public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
-    workspace: string,
+    if (buildParametersContext.maxRetainedWorkspaces === 0) {
-    runId: string,
+      return false;
-    buildParametersContext: BuildParameters,
-  ) {
-    if (!CloudRunnerOptions.retainWorkspaces) {
-      return;
     }
 
-    try {
+    if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
-      if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
+      const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
-        const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
+      CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
+      for (const element of workspaces) {
+        const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
         CloudRunnerLogger.log(
-          `run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`,
+          `run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
         );
-        for (const element of workspaces) {
-          await new Promise((promise) => setTimeout(promise, 1000));
-          const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
-          CloudRunnerLogger.log(`run agent: ${runId} try lock workspace: ${element} result: ${lockResult}`);
 
-          if (lockResult) {
+        if (lockResult) {
-            CloudRunner.lockedWorkspace = element;
+          return true;
-
-            return true;
-          }
         }
       }
-    } catch {
-      return;
     }
 
-    const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
+    if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
+      workspace = SharedWorkspaceLocking.NewWorkspaceName();
+      CloudRunner.lockedWorkspace = workspace;
+    }
+
+    const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
+    const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
     CloudRunnerLogger.log(
-      `run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`,
+      `run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
     );
 
-    return createResult;
+    return createResult && lockResult;
   }
 
   public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
-    return (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
+    return (
+      (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
+        .length > 0
+    );
   }
   public static async HasWorkspaceLock(
     workspace: string,
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
+    const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
-      return false;
-    }
-    const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
       .map((x) => {
         return {
           name: x,
@@ -115,14 +130,11 @@ export class SharedWorkspaceLocking {
     const result: string[] = [];
     const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
     for (const element of workspaces) {
-      await new Promise((promise) => setTimeout(promise, 1500));
       const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
       const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
+      CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
       if (!isLocked && isBelowMax) {
         result.push(element);
-        CloudRunnerLogger.log(`workspace ${element} is free`);
-      } else {
-        CloudRunnerLogger.log(`workspace ${element} is NOT free ${!isLocked} ${isBelowMax}`);
       }
     }
 
@@ -171,60 +183,49 @@ export class SharedWorkspaceLocking {
 
     return (
       await SharedWorkspaceLocking.ReadLines(
-        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
     )
       .map((x) => x.replace(`/`, ``))
-      .filter((x) => x.includes(`_workspace`))
+      .filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
       .map((x) => Number(x))[0];
   }
 
   public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
     if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
-      return false;
+      throw new Error(`workspace doesn't exist ${workspace}`);
     }
     const files = await SharedWorkspaceLocking.ReadLines(
-      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
     );
 
-    const workspaceFileDoesNotExists =
-      files.filter((x) => {
-        return x.includes(`_workspace`);
-      }).length === 0;
-
     const lockFilesExist =
       files.filter((x) => {
-        return x.includes(`_lock`);
+        return x.includes(workspace) && x.endsWith(`_lock`);
       }).length > 0;
 
-    return workspaceFileDoesNotExists || lockFilesExist;
+    return lockFilesExist;
   }
 
-  public static async CreateWorkspace(
+  public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
-    workspace: string,
+    if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
-    buildParametersContext: BuildParameters,
+      throw new Error(`${workspace} already exists`);
-    lockId: string = ``,
-  ): Promise<boolean> {
-    if (lockId !== ``) {
-      await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
     }
     const timestamp = Date.now();
-    const file = `${timestamp}_workspace`;
+    const file = `${timestamp}_${workspace}_workspace`;
     fs.writeFileSync(file, '');
     await CloudRunnerSystem.Run(
-      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
     fs.rmSync(file);
 
-    const workspaces = await SharedWorkspaceLocking.ReadLines(
+    const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
-      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
-    );
 
     CloudRunnerLogger.log(`All workspaces ${workspaces}`);
     if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
-      CloudRunnerLogger.log(`Workspace is below max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
+      CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
       await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
 
       return false;
@@ -238,16 +239,30 @@ export class SharedWorkspaceLocking {
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    const file = `${Date.now()}_${runId}_lock`;
+    const existingWorkspace = workspace.endsWith(`_workspace`);
+    const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
+    const file = `${Date.now()}_${runId}_${ending}_lock`;
     fs.writeFileSync(file, '');
     await CloudRunnerSystem.Run(
-      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
     fs.rmSync(file);
 
-    return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+    const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+
+    if (hasLock) {
+      CloudRunner.lockedWorkspace = workspace;
+    } else {
+      await CloudRunnerSystem.Run(
+        `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
+        false,
+        true,
+      );
+    }
+
+    return hasLock;
   }
 
   public static async ReleaseWorkspace(
@@ -255,31 +270,29 @@ export class SharedWorkspaceLocking {
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    const file = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) =>
+    const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
-      x.includes(`_${runId}_lock`),
+    const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
-    );
+    CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
     CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
-    CloudRunnerLogger.log(
+    CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
-    );
     await CloudRunnerSystem.Run(
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
 
-    return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+    return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
   }
 
   public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
     await CloudRunnerSystem.Run(
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`,
+      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
       false,
       true,
     );
   }
 
-  private static async ReadLines(command: string): Promise<string[]> {
+  public static async ReadLines(command: string): Promise<string[]> {
     return CloudRunnerSystem.RunAndReadLines(command);
   }
 }

src/model/cloud-runner/services/core/task-parameter-serializer.ts

@@ -0,0 +1,174 @@
+import BuildParameters from '../../../build-parameters';
+import Input from '../../../input';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
+import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
+import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
+import { CommandHookService } from '../hooks/command-hook-service';
+
+export class TaskParameterSerializer {
+  static readonly blockedParameterNames: Set<string> = new Set([
+    '0',
+    'length',
+    'prototype',
+    '',
+    'unityVersion',
+    'CACHE_UNITY_INSTALLATION_ON_MAC',
+    'RUNNER_TEMP_PATH',
+    'NAME',
+    'CUSTOM_JOB',
+  ]);
+  public static createCloudRunnerEnvironmentVariables(
+    buildParameters: BuildParameters,
+  ): CloudRunnerEnvironmentVariable[] {
+    const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
+      [
+        ...[
+          { name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
+          { name: 'UNITY_VERSION', value: buildParameters.editorVersion },
+          { name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
+        ],
+        ...TaskParameterSerializer.serializeFromObject(buildParameters),
+        ...TaskParameterSerializer.serializeInput(),
+        ...TaskParameterSerializer.serializeCloudRunnerOptions(),
+        ...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
+      ]
+        .filter(
+          (x) =>
+            !TaskParameterSerializer.blockedParameterNames.has(x.name) &&
+            x.value !== '' &&
+            x.value !== undefined &&
+            x.value !== `undefined`,
+        )
+        .map((x) => {
+          x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
+          x.value = `${x.value}`;
+
+          return x;
+        }),
+      (item: CloudRunnerEnvironmentVariable) => item.name,
+    );
+
+    return result;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) {
+    const seen: { [key: string]: boolean } = {};
+
+    return a.filter(function (item) {
+      const k = key(item);
+
+      return seen.hasOwnProperty(k) ? false : (seen[k] = true);
+    });
+  }
+
+  public static readBuildParameterFromEnvironment(): BuildParameters {
+    const buildParameters = new BuildParameters();
+    const keys = [
+      ...new Set(
+        Object.getOwnPropertyNames(process.env)
+          .filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
+          .map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
+      ),
+    ];
+
+    for (const element of keys) {
+      if (element !== `customJob`) {
+        buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
+      }
+    }
+
+    return buildParameters;
+  }
+
+  private static serializeInput() {
+    return TaskParameterSerializer.serializeFromType(Input);
+  }
+
+  private static serializeCloudRunnerOptions() {
+    return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
+  }
+
+  public static ToEnvVarFormat(input: string): string {
+    return CloudRunnerOptions.ToEnvVarFormat(input);
+  }
+
+  public static UndoEnvVarFormat(element: string): string {
+    return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
+  }
+
+  private static camelize(string: string) {
+    return TaskParameterSerializer.uncapitalizeFirstLetter(
+      string
+        .replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
+          return index === 0 ? word.toLowerCase() : word.toUpperCase();
+        })
+        .replace(/\s+/g, ''),
+    );
+  }
+
+  private static uncapitalizeFirstLetter(string: string) {
+    return string.charAt(0).toLowerCase() + string.slice(1);
+  }
+
+  private static serializeFromObject(buildParameters: any) {
+    const array: any[] = [];
+    const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
+    for (const element of keys) {
+      array.push({
+        name: TaskParameterSerializer.ToEnvVarFormat(element),
+        value: buildParameters[element],
+      });
+    }
+
+    return array;
+  }
+
+  private static serializeFromType(type: any) {
+    const array: any[] = [];
+    const input = CloudRunnerOptionsReader.GetProperties();
+    for (const element of input) {
+      if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
+        array.push({
+          name: element,
+          value: `${type[element]}`,
+        });
+      }
+    }
+
+    return array;
+  }
+
+  public static readDefaultSecrets(): CloudRunnerSecret[] {
+    let array = new Array();
+    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
+    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
+    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
+    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
+    array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');
+
+    return array;
+  }
+
+  private static getValue(key: string) {
+    return CloudRunnerQueryOverride.queryOverrides !== undefined &&
+      CloudRunnerQueryOverride.queryOverrides[key] !== undefined
+      ? CloudRunnerQueryOverride.queryOverrides[key]
+      : process.env[key];
+  }
+
+  private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] {
+    const value = TaskParameterSerializer.getValue(key);
+    if (value !== undefined && value !== '' && value !== 'null') {
+      array.push({
+        ParameterKey: key,
+        EnvironmentVariable: key,
+        ParameterValue: value,
+      });
+    }
+
+    return array;
+  }
+}

src/model/cloud-runner/services/custom-step.ts

@@ -1,9 +0,0 @@
-import CloudRunnerSecret from './cloud-runner-secret';
-
-export class CustomStep {
-  public commands;
-  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
-  public name;
-  public image: string = `ubuntu`;
-  public hook!: string;
-}

src/model/cloud-runner/services/follow-log-stream-service.ts

@@ -1,37 +0,0 @@
-import CloudRunnerLogger from './cloud-runner-logger';
-import * as core from '@actions/core';
-import CloudRunner from '../cloud-runner';
-import { CloudRunnerStatics } from '../cloud-runner-statics';
-import GitHub from '../../github';
-
-export class FollowLogStreamService {
-  public static handleIteration(message, shouldReadLogs, shouldCleanup, output) {
-    if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
-      CloudRunnerLogger.log('End of log transmission received');
-      shouldReadLogs = false;
-    } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
-      GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
-      core.warning('LIBRARY NOT FOUND!');
-      core.setOutput('library-found', 'false');
-    } else if (message.includes('Build succeeded')) {
-      GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
-      core.setOutput('build-result', 'success');
-    } else if (message.includes('Build fail')) {
-      GitHub.updateGitHubCheck(`Build failed`, `Build failed`);
-      core.setOutput('build-result', 'failed');
-      core.setFailed('unity build failed');
-      core.error('BUILD FAILED!');
-    } else if (CloudRunner.buildParameters.cloudRunnerDebug && message.includes(': Listening for Jobs')) {
-      core.setOutput('cloud runner stop watching', 'true');
-      shouldReadLogs = false;
-      shouldCleanup = false;
-      core.warning('cloud runner stop watching');
-    }
-    if (CloudRunner.buildParameters.cloudRunnerDebug) {
-      output += `${message}\n`;
-    }
-    CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
-
-    return { shouldReadLogs, shouldCleanup, output };
-  }
-}

src/model/cloud-runner/services/hooks/command-hook-service.ts

@@ -0,0 +1,118 @@
+import { BuildParameters, Input } from '../../..';
+import YAML from 'yaml';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
+import path from 'node:path';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
+import * as fs from 'node:fs';
+import CloudRunnerLogger from '../core/cloud-runner-logger';
+import { CommandHook } from './command-hook';
+
+// import CloudRunnerLogger from './cloud-runner-logger';
+
+export class CommandHookService {
+  public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
+    const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
+    CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
+
+    return `echo "---"
+echo "start cloud runner init"
+${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
+echo "start of cloud runner job"
+${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
+${commands}
+${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
+echo "end of cloud runner job"
+echo "---${buildParameters.logId}"`;
+  }
+
+  public static getHooks(customCommandHooks: string): CommandHook[] {
+    const experimentHooks = customCommandHooks;
+    let output = new Array<CommandHook>();
+    if (experimentHooks && experimentHooks !== '') {
+      try {
+        output = YAML.parse(experimentHooks);
+      } catch (error) {
+        throw error;
+      }
+    }
+
+    return [
+      ...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
+      ...CommandHookService.GetCustomHooksFromFiles(`before`),
+      ...CommandHookService.GetCustomHooksFromFiles(`after`),
+    ];
+  }
+
+  static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
+    const results: CommandHook[] = [];
+
+    // RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
+    try {
+      const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
+      const files = fs.readdirSync(gameCiCustomHooksPath);
+      for (const file of files) {
+        if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
+          continue;
+        }
+        const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
+        const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
+        if (fileContentsObject.hook.includes(hookLifecycle)) {
+          results.push(fileContentsObject);
+        }
+      }
+    } catch (error) {
+      RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
+    }
+
+    // RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
+
+    return results;
+  }
+
+  private static ConvertYamlSecrets(object: CommandHook) {
+    if (object.secrets === undefined) {
+      object.secrets = [];
+
+      return;
+    }
+    object.secrets = object.secrets.map((x: any) => {
+      return {
+        ParameterKey: x.name,
+        EnvironmentVariable: Input.ToEnvVarFormat(x.name),
+        ParameterValue: x.value,
+      };
+    });
+  }
+
+  public static ParseHooks(hooks: string): CommandHook[] {
+    if (hooks === '') {
+      return [];
+    }
+
+    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
+
+    // CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
+
+    // }
+    const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
+    const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
+    for (const hook of object) {
+      CommandHookService.ConvertYamlSecrets(hook);
+      if (hook.secrets === undefined) {
+        hook.secrets = [];
+      }
+    }
+    if (object === undefined) {
+      throw new Error(`Failed to parse ${hooks}`);
+    }
+
+    return object;
+  }
+
+  public static getSecrets(hooks: any) {
+    const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
+
+    // eslint-disable-next-line unicorn/no-array-reduce
+    return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
+  }
+}

src/model/cloud-runner/services/hooks/command-hook.ts

@@ -0,0 +1,9 @@
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
+
+export class CommandHook {
+  public commands: string[] = new Array<string>();
+  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
+  public name!: string;
+  public hook!: string[];
+  public step!: string[];
+}

src/model/cloud-runner/services/hooks/container-hook-service.ts

@@ -1,31 +1,28 @@
 import YAML from 'yaml';
-import CloudRunner from '../cloud-runner';
+import CloudRunner from '../../cloud-runner';
 import * as core from '@actions/core';
-import { CustomWorkflow } from '../workflows/custom-workflow';
+import { CustomWorkflow } from '../../workflows/custom-workflow';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
-import path from 'path';
+import path from 'node:path';
-import * as fs from 'fs';
+import fs from 'node:fs';
-import Input from '../../input';
+import Input from '../../../input';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
-import CloudRunnerLogger from './cloud-runner-logger';
+import { ContainerHook as ContainerHook } from './container-hook';
-import { CustomStep } from './custom-step';
+import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
 
-export class CloudRunnerCustomSteps {
+export class ContainerHookService {
-  static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
+  static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
-    const results: CustomStep[] = [];
+    const results: ContainerHook[] = [];
-    RemoteClientLogger.log(
-      `GetCustomStepFiles: ${hookLifecycle} CustomStepFiles: ${CloudRunnerOptions.customStepFiles}`,
-    );
     try {
-      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `steps`);
+      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
       const files = fs.readdirSync(gameCiCustomStepsPath);
       for (const file of files) {
-        if (!CloudRunnerOptions.customStepFiles.includes(file.replace(`.yaml`, ``))) {
+        if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
-          RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
+          // RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
           continue;
         }
         const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
-        const fileContentsObject = CloudRunnerCustomSteps.ParseSteps(fileContents)[0];
+        const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
         if (fileContentsObject.hook === hookLifecycle) {
           results.push(fileContentsObject);
         }
@@ -33,9 +30,10 @@ export class CloudRunnerCustomSteps {
     } catch (error) {
       RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
     }
-    RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
 
-    const builtInCustomSteps: CustomStep[] = CloudRunnerCustomSteps.ParseSteps(
+    // RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
+
+    const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
       `- name: aws-s3-upload-build
   image: amazon/aws-cli
   hook: after
@@ -44,12 +42,12 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
     aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
-      } s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
+      } s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
     rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
   secrets:
   - name: awsAccessKeyId
@@ -64,19 +62,20 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
+    mkdir -p /data/cache/$CACHE_KEY/build/
     aws s3 cp s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       } /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
   secrets:
-    - name: awsAccessKeyId
+    - name: AWS_ACCESS_KEY_ID
-    - name: awsSecretAccessKey
+    - name: AWS_SECRET_ACCESS_KEY
-    - name: awsDefaultRegion
+    - name: AWS_DEFAULT_REGION
     - name: BUILD_GUID_TARGET
 - name: steam-deploy-client
   image: steamcmd/steamcmd
@@ -122,19 +121,19 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
     aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/lfs
     rm -r /data/cache/$CACHE_KEY/lfs
     aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/Library
     rm -r /data/cache/$CACHE_KEY/Library
   secrets:
-  - name: awsAccessKeyId
+  - name: AWS_ACCESS_KEY_ID
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
-  - name: awsSecretAccessKey
+  - name: AWS_SECRET_ACCESS_KEY
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
-  - name: awsDefaultRegion
+  - name: AWS_DEFAULT_REGION
     value: ${process.env.AWS_REGION || ``}
 - name: aws-s3-pull-cache
   image: amazon/aws-cli
@@ -143,30 +142,32 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    mkdir -p /data/cache/$CACHE_KEY/Library/
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
+    mkdir -p /data/cache/$CACHE_KEY/lfs/
-    BUCKET1="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
+    BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
     aws s3 ls $BUCKET1 || true
     OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
     aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
-    BUCKET2="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
+    BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
     aws s3 ls $BUCKET2 || true
     OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
     aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
   secrets:
-  - name: awsAccessKeyId
+  - name: AWS_ACCESS_KEY_ID
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
-  - name: awsSecretAccessKey
+  - name: AWS_SECRET_ACCESS_KEY
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
-  - name: awsDefaultRegion
+  - name: AWS_DEFAULT_REGION
     value: ${process.env.AWS_REGION || ``}
 - name: debug-cache
   image: ubuntu
   hook: after
   commands: |
     apt-get update > /dev/null
-    ${CloudRunnerOptions.cloudRunnerDebugTree ? `apt-get install -y tree > /dev/null` : `#`}
+    ${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
-    ${CloudRunnerOptions.cloudRunnerDebugTree ? `tree -L 3 /data/cache` : `#`}
+    ${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
   secrets:
   - name: awsAccessKeyId
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@@ -174,21 +175,21 @@ export class CloudRunnerCustomSteps {
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
   - name: awsDefaultRegion
     value: ${process.env.AWS_REGION || ``}`,
-    ).filter((x) => CloudRunnerOptions.customStepFiles.includes(x.name) && x.hook === hookLifecycle);
+    ).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
-    if (builtInCustomSteps.length > 0) {
+    if (builtInContainerHooks.length > 0) {
-      results.push(...builtInCustomSteps);
+      results.push(...builtInContainerHooks);
     }
 
     return results;
   }
 
-  private static ConvertYamlSecrets(object) {
+  private static ConvertYamlSecrets(object: ContainerHook) {
     if (object.secrets === undefined) {
       object.secrets = [];
 
       return;
     }
-    object.secrets = object.secrets.map((x) => {
+    object.secrets = object.secrets.map((x: { [key: string]: any }) => {
       return {
         ParameterKey: x.name,
         EnvironmentVariable: Input.ToEnvVarFormat(x.name),
@@ -197,21 +198,21 @@ export class CloudRunnerCustomSteps {
     });
   }
 
-  public static ParseSteps(steps: string): CustomStep[] {
+  public static ParseContainerHooks(steps: string): ContainerHook[] {
     if (steps === '') {
       return [];
     }
     const isArray = steps.replace(/\s/g, ``)[0] === `-`;
-    const object: CustomStep[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
+    const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
     for (const step of object) {
-      CloudRunnerCustomSteps.ConvertYamlSecrets(step);
+      ContainerHookService.ConvertYamlSecrets(step);
       if (step.secrets === undefined) {
         step.secrets = [];
       } else {
         for (const secret of step.secrets) {
           if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
             if (CloudRunner.buildParameters?.cloudRunnerDebug) {
-              CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
+              // CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
             }
             secret.ParameterValue = process.env[secret.ParameterKey] || ``;
           }
@@ -228,16 +229,16 @@ export class CloudRunnerCustomSteps {
     return object;
   }
 
-  static async RunPostBuildSteps(cloudRunnerStepState) {
+  static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
     let output = ``;
-    const steps: CustomStep[] = [
+    const steps: ContainerHook[] = [
-      ...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.postBuildSteps),
+      ...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
-      ...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`after`),
+      ...ContainerHookService.GetContainerHooksFromFiles(`after`),
     ];
 
     if (steps.length > 0) {
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
-      output += await CustomWorkflow.runCustomJob(
+      output += await CustomWorkflow.runContainerJob(
         steps,
         cloudRunnerStepState.environment,
         cloudRunnerStepState.secrets,
@@ -247,16 +248,16 @@ export class CloudRunnerCustomSteps {
 
     return output;
   }
-  static async RunPreBuildSteps(cloudRunnerStepState) {
+  static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
     let output = ``;
-    const steps: CustomStep[] = [
+    const steps: ContainerHook[] = [
-      ...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.preBuildSteps),
+      ...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
-      ...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`),
+      ...ContainerHookService.GetContainerHooksFromFiles(`before`),
     ];
 
     if (steps.length > 0) {
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
-      output += await CustomWorkflow.runCustomJob(
+      output += await CustomWorkflow.runContainerJob(
         steps,
         cloudRunnerStepState.environment,
         cloudRunnerStepState.secrets,

src/model/cloud-runner/services/hooks/container-hook.ts

@@ -0,0 +1,9 @@
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
+
+export class ContainerHook {
+  public commands!: string;
+  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
+  public name!: string;
+  public image: string = `ubuntu`;
+  public hook!: string;
+}

src/model/cloud-runner/services/lfs-hashing.ts

@@ -1,47 +0,0 @@
-import path from 'path';
-import { CloudRunnerFolders } from './cloud-runner-folders';
-import { CloudRunnerSystem } from './cloud-runner-system';
-import fs from 'fs';
-import { assert } from 'console';
-import { Cli } from '../../cli/cli';
-import { CliFunction } from '../../cli/cli-functions-repository';
-
-export class LfsHashing {
-  public static async createLFSHashFiles() {
-    try {
-      await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
-      await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
-      assert(fs.existsSync(`.lfs-assets-guid-sum`));
-      assert(fs.existsSync(`.lfs-assets-guid`));
-      const lfsHashes = {
-        lfsGuid: fs
-          .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
-          .replace(/\n/g, ``),
-        lfsGuidSum: fs
-          .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
-          .replace('  .lfs-assets-guid', '')
-          .replace(/\n/g, ``),
-      };
-
-      return lfsHashes;
-    } catch (error) {
-      throw error;
-    }
-  }
-  public static async hashAllFiles(folder: string) {
-    const startPath = process.cwd();
-    process.chdir(folder);
-    const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
-      .replace(/\n/g, '')
-      .split(` `)[0];
-    process.chdir(startPath);
-
-    return result;
-  }
-
-  @CliFunction(`hash`, `hash all folder contents`)
-  static async hash() {
-    const folder = Cli.options['cachePushFrom'];
-    LfsHashing.hashAllFiles(folder);
-  }
-}

src/model/cloud-runner/services/task-parameter-serializer.ts

@@ -1,171 +0,0 @@
-import { Input } from '../..';
-import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
-import { CloudRunnerCustomHooks } from './cloud-runner-custom-hooks';
-import CloudRunnerSecret from './cloud-runner-secret';
-import CloudRunnerQueryOverride from './cloud-runner-query-override';
-import CloudRunnerOptionsReader from './cloud-runner-options-reader';
-import BuildParameters from '../../build-parameters';
-import CloudRunnerOptions from '../cloud-runner-options';
-import * as core from '@actions/core';
-
-export class TaskParameterSerializer {
-  static readonly blocked = new Set(['0', 'length', 'prototype', '', 'unityVersion']);
-  public static createCloudRunnerEnvironmentVariables(
-    buildParameters: BuildParameters,
-  ): CloudRunnerEnvironmentVariable[] {
-    const result = this.uniqBy(
-      [
-        {
-          name: 'ContainerMemory',
-          value: buildParameters.cloudRunnerMemory,
-        },
-        {
-          name: 'ContainerCpu',
-          value: buildParameters.cloudRunnerCpu,
-        },
-        {
-          name: 'BUILD_TARGET',
-          value: buildParameters.targetPlatform,
-        },
-        ...TaskParameterSerializer.serializeFromObject(buildParameters),
-        ...TaskParameterSerializer.readInput(),
-        ...CloudRunnerCustomHooks.getSecrets(CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks)),
-      ]
-        .filter(
-          (x) =>
-            !TaskParameterSerializer.blocked.has(x.name) &&
-            x.value !== '' &&
-            x.value !== undefined &&
-            x.name !== `CUSTOM_JOB` &&
-            x.name !== `GAMECI_CUSTOM_JOB` &&
-            x.value !== `undefined`,
-        )
-        .map((x) => {
-          x.name = TaskParameterSerializer.ToEnvVarFormat(x.name);
-          x.value = `${x.value}`;
-
-          if (buildParameters.cloudRunnerDebug && Number(x.name) === Number.NaN) {
-            core.info(`[ERROR] found a number in task param serializer ${JSON.stringify(x)}`);
-          }
-
-          return x;
-        }),
-      (item) => item.name,
-    );
-
-    return result;
-  }
-
-  static uniqBy(a, key) {
-    const seen = {};
-
-    return a.filter(function (item) {
-      const k = key(item);
-
-      return seen.hasOwnProperty(k) ? false : (seen[k] = true);
-    });
-  }
-
-  public static readBuildParameterFromEnvironment(): BuildParameters {
-    const buildParameters = new BuildParameters();
-    const keys = [
-      ...new Set(
-        Object.getOwnPropertyNames(process.env)
-          .filter((x) => !this.blocked.has(x) && x.startsWith('GAMECI_'))
-          .map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
-      ),
-    ];
-
-    for (const element of keys) {
-      if (element !== `customJob`) {
-        buildParameters[element] = process.env[`GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`];
-      }
-    }
-
-    return buildParameters;
-  }
-
-  private static readInput() {
-    return TaskParameterSerializer.serializeFromType(Input);
-  }
-
-  public static ToEnvVarFormat(input): string {
-    return CloudRunnerOptions.ToEnvVarFormat(input);
-  }
-
-  public static UndoEnvVarFormat(element): string {
-    return this.camelize(element.replace('GAMECI_', '').toLowerCase().replace(/_+/g, ' '));
-  }
-
-  private static camelize(string) {
-    return string
-      .replace(/^\w|[A-Z]|\b\w/g, function (word, index) {
-        return index === 0 ? word.toLowerCase() : word.toUpperCase();
-      })
-      .replace(/\s+/g, '');
-  }
-
-  private static serializeFromObject(buildParameters) {
-    const array: any[] = [];
-    const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blocked.has(x));
-    for (const element of keys) {
-      array.push(
-        {
-          name: `GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`,
-          value: buildParameters[element],
-        },
-        {
-          name: element,
-          value: buildParameters[element],
-        },
-      );
-    }
-
-    return array;
-  }
-
-  private static serializeFromType(type) {
-    const array: any[] = [];
-    const input = CloudRunnerOptionsReader.GetProperties();
-    for (const element of input) {
-      if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
-        array.push({
-          name: element,
-          value: `${type[element]}`,
-        });
-      }
-    }
-
-    return array;
-  }
-
-  public static readDefaultSecrets(): CloudRunnerSecret[] {
-    let array = new Array();
-    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
-    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
-    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
-    array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
-    array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');
-
-    return array;
-  }
-  private static getValue(key) {
-    return CloudRunnerQueryOverride.queryOverrides !== undefined &&
-      CloudRunnerQueryOverride.queryOverrides[key] !== undefined
-      ? CloudRunnerQueryOverride.queryOverrides[key]
-      : process.env[key];
-  }
-  s;
-  private static tryAddInput(array, key): CloudRunnerSecret[] {
-    const value = TaskParameterSerializer.getValue(key);
-    if (value !== undefined && value !== '' && value !== 'null') {
-      array.push({
-        ParameterKey: key,
-        EnvironmentVariable: key,
-        ParameterValue: value,
-      });
-    }
-
-    return array;
-  }
-}

src/model/cloud-runner/services/utility/lfs-hashing.ts

@@ -0,0 +1,43 @@
+import path from 'node:path';
+import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
+import { CloudRunnerSystem } from '../core/cloud-runner-system';
+import fs from 'node:fs';
+import { Cli } from '../../../cli/cli';
+import { CliFunction } from '../../../cli/cli-functions-repository';
+
+export class LfsHashing {
+  public static async createLFSHashFiles() {
+    await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
+    await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
+    const lfsHashes = {
+      lfsGuid: fs
+        .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
+        .replace(/\n/g, ``),
+      lfsGuidSum: fs
+        .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
+        .replace('  .lfs-assets-guid', '')
+        .replace(/\n/g, ``),
+    };
+
+    return lfsHashes;
+  }
+  public static async hashAllFiles(folder: string) {
+    const startPath = process.cwd();
+    process.chdir(folder);
+    const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
+      .replace(/\n/g, '')
+      .split(` `)[0];
+    process.chdir(startPath);
+
+    return result;
+  }
+
+  @CliFunction(`hash`, `hash all folder contents`)
+  static async hash() {
+    if (!Cli.options) {
+      return;
+    }
+    const folder = Cli.options['cachePushFrom'];
+    LfsHashing.hashAllFiles(folder);
+  }
+}

src/model/cloud-runner/tests/cloud-runner-async-workflow.test.ts

@@ -2,10 +2,11 @@ import { BuildParameters, ImageTag } from '../..';
 import CloudRunner from '../cloud-runner';
 import UnityVersioning from '../../unity-versioning';
 import { Cli } from '../../cli/cli';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 import setups from './cloud-runner-suite.test';
+import { OptionValues } from 'commander';
 
-async function CreateParameters(overrides) {
+async function CreateParameters(overrides: OptionValues | undefined) {
   if (overrides) Cli.options = overrides;
 
   return BuildParameters.create();
@@ -14,7 +15,7 @@ describe('Cloud Runner Async Workflows', () => {
   setups();
   it('Responds', () => {});
 
-  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
     it('Async Workflows', async () => {
       // Setup parameters
       const buildParameter = await CreateParameters({

src/model/cloud-runner/tests/cloud-runner-caching.test.ts

@@ -1,17 +1,18 @@
-import fs from 'fs';
+import fs from 'node:fs';
-import path from 'path';
+import path from 'node:path';
 import BuildParameters from '../../build-parameters';
 import { Cli } from '../../cli/cli';
 import UnityVersioning from '../../unity-versioning';
 import CloudRunner from '../cloud-runner';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import { Caching } from '../remote-client/caching';
 import { v4 as uuidv4 } from 'uuid';
 import GitHub from '../../github';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 describe('Cloud Runner (Remote Client) Caching', () => {
   it('responds', () => {});
-  if (process.platform === 'linux') {
+  if (CloudRunnerOptions.providerStrategy === `local-docker`) {
-    it.skip('Simple caching works', async () => {
+    it('Simple caching works', async () => {
       Cli.options = {
         versioning: 'None',
         projectPath: 'test-project',

src/model/cloud-runner/tests/cloud-runner-environment-serializer.test.ts

@@ -1,45 +0,0 @@
-import { BuildParameters } from '../..';
-import { TaskParameterSerializer } from '../services/task-parameter-serializer';
-import UnityVersioning from '../../unity-versioning';
-import { Cli } from '../../cli/cli';
-import GitHub from '../../github';
-import setups from './cloud-runner-suite.test';
-
-async function CreateParameters(overrides) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-  const originalValue = GitHub.githubInputEnabled;
-  GitHub.githubInputEnabled = false;
-  const results = await BuildParameters.create();
-  GitHub.githubInputEnabled = originalValue;
-  delete Cli.options;
-
-  return results;
-}
-describe('Cloud Runner Environment Serializer', () => {
-  setups();
-  const testSecretName = 'testSecretName';
-  const testSecretValue = 'testSecretValue';
-  it('Cloud Runner Parameter Serialization', async () => {
-    // Setup parameters
-    const buildParameter = await CreateParameters({
-      versioning: 'None',
-      projectPath: 'test-project',
-      unityVersion: UnityVersioning.read('test-project'),
-      customJob: `
-      - name: 'step 1'
-        image: 'alpine'
-        commands: 'printenv'
-        secrets:
-          - name: '${testSecretName}'
-            value: '${testSecretValue}'
-      `,
-    });
-
-    const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
-    expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
-    const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
-    expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
-  });
-});

src/model/cloud-runner/tests/cloud-runner-environment.test.ts

@@ -1,19 +1,26 @@
-import { BuildParameters, ImageTag } from '../..';
+import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
-import CloudRunner from '../cloud-runner';
+import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
-import Input from '../../input';
-import { CloudRunnerStatics } from '../cloud-runner-statics';
-import { TaskParameterSerializer } from '../services/task-parameter-serializer';
 import UnityVersioning from '../../unity-versioning';
 import { Cli } from '../../cli/cli';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import GitHub from '../../github';
-import CloudRunnerOptions from '../cloud-runner-options';
 import setups from './cloud-runner-suite.test';
+import { CloudRunnerStatics } from '../options/cloud-runner-statics';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 
-async function CreateParameters(overrides) {
+async function CreateParameters(overrides: any) {
-  if (overrides) Cli.options = overrides;
+  if (overrides) {
+    Cli.options = overrides;
+  }
+  const originalValue = GitHub.githubInputEnabled;
+  GitHub.githubInputEnabled = false;
+  const results = await BuildParameters.create();
+  GitHub.githubInputEnabled = originalValue;
+  delete Cli.options;
 
-  return BuildParameters.create();
+  return results;
 }
+
 describe('Cloud Runner Sync Environments', () => {
   setups();
   const testSecretName = 'testSecretName';
@@ -61,7 +68,7 @@ describe('Cloud Runner Sync Environments', () => {
           return x;
         })
         .filter((element) => {
-          return !['UNITY_LICENSE', 'CUSTOM_JOB'].includes(element.name);
+          return !['UNITY_LICENSE', 'UNITY_LICENSE', 'CUSTOM_JOB', 'CUSTOM_JOB'].includes(element.name);
         });
       const newLinePurgedFile = file
         .replace(/\s+/g, '')
@@ -75,3 +82,30 @@ describe('Cloud Runner Sync Environments', () => {
     }, 1_000_000_000);
   }
 });
+
+describe('Cloud Runner Environment Serializer', () => {
+  setups();
+  const testSecretName = 'testSecretName';
+  const testSecretValue = 'testSecretValue';
+  it('Cloud Runner Parameter Serialization', async () => {
+    // Setup parameters
+    const buildParameter = await CreateParameters({
+      versioning: 'None',
+      projectPath: 'test-project',
+      unityVersion: UnityVersioning.read('test-project'),
+      customJob: `
+      - name: 'step 1'
+        image: 'alpine'
+        commands: 'printenv'
+        secrets:
+          - name: '${testSecretName}'
+            value: '${testSecretValue}'
+      `,
+    });
+
+    const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
+    expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
+    const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
+    expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
+  });
+});

src/model/cloud-runner/tests/cloud-runner-hooks.test.ts

@@ -0,0 +1,114 @@
+import CloudRunner from '../cloud-runner';
+import { BuildParameters, ImageTag } from '../..';
+import UnityVersioning from '../../unity-versioning';
+import { Cli } from '../../cli/cli';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import { v4 as uuidv4 } from 'uuid';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import setups from './cloud-runner-suite.test';
+import { ContainerHookService } from '../services/hooks/container-hook-service';
+import { CommandHookService } from '../services/hooks/command-hook-service';
+
+async function CreateParameters(overrides: any) {
+  if (overrides) {
+    Cli.options = overrides;
+  }
+
+  return await BuildParameters.create();
+}
+
+describe('Cloud Runner Custom Hooks And Steps', () => {
+  it('Responds', () => {});
+  setups();
+  it('Check parsing and reading of steps', async () => {
+    const yamlString = `hook: before
+commands: echo "test"`;
+    const yamlString2 = `- hook: before
+  commands: echo "test"`;
+    const overrides = {
+      versioning: 'None',
+      projectPath: 'test-project',
+      unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+      targetPlatform: 'StandaloneLinux64',
+      cacheKey: `test-case-${uuidv4()}`,
+    };
+    CloudRunner.setup(await CreateParameters(overrides));
+    const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
+    const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
+
+    CloudRunnerLogger.log(yamlString);
+    CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
+
+    expect(stringObject.length).toBe(1);
+    expect(stringObject[0].hook).toBe(`before`);
+    expect(stringObject2.length).toBe(1);
+    expect(stringObject2[0].hook).toBe(`before`);
+
+    const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
+    CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
+  });
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
+    it('Should be 1 before and 1 after hook', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      await CloudRunner.setup(buildParameter2);
+      const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
+      const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
+      expect(beforeHooks).toHaveLength(1);
+      expect(afterHooks).toHaveLength(1);
+    });
+    it('Should be 1 before and 1 after step', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      await CloudRunner.setup(buildParameter2);
+      const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
+      const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
+      expect(beforeSteps).toHaveLength(1);
+      expect(afterSteps).toHaveLength(1);
+    });
+    it('Run build once - check for pre and post custom hooks run contents', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      const baseImage2 = new ImageTag(buildParameter2);
+      const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
+      CloudRunnerLogger.log(`run 2 succeeded`);
+
+      const buildContainsBuildSucceeded = results2.includes('Build succeeded');
+      const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
+      const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
+
+      const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
+      const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
+
+      expect(buildContainsBuildSucceeded).toBeTruthy();
+      expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
+      expect(buildContainsPostBuildHookRunMessage).toBeTruthy();
+      expect(buildContainsPreBuildStepMessage).toBeTruthy();
+      expect(buildContainsPostBuildStepMessage).toBeTruthy();
+    }, 1_000_000_000);
+  }
+});