Cloud Runner Develop - 1.0 R.C 1 (#437)

Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
2022-11-07 20:41:00 +00:00
committed by GitHub
parent 96555a0945
commit 4cca069ebb
76 changed files with 5325 additions and 2023 deletions

View File

@@ -0,0 +1,45 @@
import { BuildParameters } from '../..';
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
import setups from './cloud-runner-suite.test';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
const originalValue = GitHub.githubInputEnabled;
GitHub.githubInputEnabled = false;
const results = await BuildParameters.create();
GitHub.githubInputEnabled = originalValue;
delete Cli.options;
return results;
}
describe('Cloud Runner Environment Serializer', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Cloud Runner Parameter Serialization', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'alpine'
commands: 'printenv'
secrets:
- name: '${testSecretName}'
value: '${testSecretValue}'
`,
});
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
});
});

View File

@@ -0,0 +1,58 @@
import fs from 'fs';
import path from 'path';
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import UnityVersioning from '../../unity-versioning';
import CloudRunner from '../cloud-runner';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github';
describe('Cloud Runner (Remote Client) Caching', () => {
it('responds', () => {});
if (process.platform === 'linux') {
it.skip('Simple caching works', async () => {
Cli.options = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
GitHub.githubInputEnabled = false;
const buildParameter = await BuildParameters.create();
CloudRunner.buildParameters = buildParameter;
// Create test folder
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
fs.mkdirSync(testFolder);
// Create cache folder
const cacheFolder = path.resolve(__dirname, `cache-${Cli.options.cacheKey}`);
fs.mkdirSync(cacheFolder);
// Add test file to test folders
fs.writeFileSync(path.resolve(testFolder, 'test.txt'), Cli.options.cacheKey);
await Caching.PushToCache(cacheFolder, testFolder, `${Cli.options.cacheKey}`);
// Delete test folder
fs.rmdirSync(testFolder, { recursive: true });
await Caching.PullFromCache(
cacheFolder.replace(/\\/g, `/`),
testFolder.replace(/\\/g, `/`),
`${Cli.options.cacheKey}`,
);
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
// Compare validity to original hash
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
Cli.options.cacheKey,
);
fs.rmdirSync(testFolder, { recursive: true });
fs.rmdirSync(cacheFolder, { recursive: true });
GitHub.githubInputEnabled = true;
delete Cli.options;
}, 1000000);
}
});

View File

@@ -0,0 +1,70 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Custom Hooks And Steps', () => {
it('Responds', () => {});
setups();
it('Check parsing and reading of steps', async () => {
const yamlString = `hook: before
commands: echo "test"`;
const yamlString2 = `- hook: before
commands: echo "test"`;
const stringObject = CloudRunnerCustomSteps.ParseSteps(yamlString);
const stringObject2 = CloudRunnerCustomSteps.ParseSteps(yamlString2);
CloudRunnerLogger.log(yamlString);
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
expect(stringObject.length).toBe(1);
expect(stringObject[0].hook).toBe(`before`);
expect(stringObject2.length).toBe(1);
expect(stringObject2[0].hook).toBe(`before`);
const getCustomStepsFromFiles = CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`);
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
});
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
it('Run build once - check for pre and post custom hooks run contents', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `my-test-step-pre-build,my-test-step-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
const build2ContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
const build2ContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
const build2ContainsPreBuildStepMessage = results2.includes('before-build step test!');
const build2ContainsPostBuildStepMessage = results2.includes('after-build step test!');
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2ContainsPreBuildHookRunMessage).toBeTruthy();
expect(build2ContainsPostBuildHookRunMessage).toBeTruthy();
expect(build2ContainsPreBuildStepMessage).toBeTruthy();
expect(build2ContainsPostBuildStepMessage).toBeTruthy();
}, 1_000_000_000);
}
});

View File

@@ -0,0 +1,72 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Caching', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug) {
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `debug-cache`,
};
if (CloudRunnerOptions.cloudRunnerCluster === `k8s`) {
overrides.customStepFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
}
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter);
const results = await CloudRunner.run(buildParameter, baseImage.toString());
const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
const buildSucceededString = 'Build succeeded';
expect(results).toContain(libraryString);
expect(results).toContain(buildSucceededString);
expect(results).not.toContain(cachePushFail);
CloudRunnerLogger.log(`run 1 succeeded`);
const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey;
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for Library',
);
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for LFS',
);
expect(build2ContainsCacheKey).toBeTruthy();
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
expect(build2NotContainsNoLibraryMessage).toBeTruthy();
}, 1_000_000_000);
}
});

View File

@@ -0,0 +1,93 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import * as fs from 'fs';
import path from 'path';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import SharedWorkspaceLocking from '../services/shared-workspace-locking';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Retain Workspace', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug) {
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
retainWorkspaces: true,
};
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter);
const results = await CloudRunner.run(buildParameter, baseImage.toString());
const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
const buildSucceededString = 'Build succeeded';
expect(results).toContain(libraryString);
expect(results).toContain(buildSucceededString);
expect(results).not.toContain(cachePushFail);
CloudRunnerLogger.log(`run 1 succeeded`);
const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey;
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for Library',
);
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for LFS',
);
expect(build2ContainsCacheKey).toBeTruthy();
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy();
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
expect(build2NotContainsNoLibraryMessage).toBeTruthy();
}, 1_000_000_000);
afterAll(async () => {
await SharedWorkspaceLocking.CleanupWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters);
if (
fs.existsSync(`./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`)
) {
CloudRunnerLogger.log(
`Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
await CloudRunnerSystem.Run(
`rm -r ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
}
});
}
});

View File

@@ -0,0 +1,46 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner pre-built S3 steps', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
expect(build2ContainsBuildSucceeded).toBeTruthy();
const results = await CloudRunnerSystem.RunAndReadLines(
`aws s3 ls s3://game-ci-test-storage/cloud-runner-cache/${buildParameter2.cacheKey}/`,
);
CloudRunnerLogger.log(results.join(`,`));
}, 1_000_000_000);
}
});

View File

@@ -0,0 +1,25 @@
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
describe('Cloud Runner', () => {
it('Responds', () => {});
});
const setups = () => {
beforeAll(() => {
GitHub.githubInputEnabled = false;
});
beforeEach(() => {
Cli.options = {};
});
afterEach(() => {
if (Cli.options !== undefined) {
delete Cli.options;
}
});
afterAll(() => {
GitHub.githubInputEnabled = true;
});
};
export default setups;

View File

@@ -0,0 +1,77 @@
import { BuildParameters, ImageTag } from '../..';
import CloudRunner from '../cloud-runner';
import Input from '../../input';
import { CloudRunnerStatics } from '../cloud-runner-statics';
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
async function CreateParameters(overrides) {
if (overrides) Cli.options = overrides;
return BuildParameters.create();
}
describe('Cloud Runner Sync Environments', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it('All build parameters sent to cloud runner as env vars', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'ubuntu'
commands: 'printenv'
secrets:
- name: '${testSecretName}'
value: '${testSecretValue}'
`,
});
const baseImage = new ImageTag(buildParameter);
// Run the job
const file = await CloudRunner.run(buildParameter, baseImage.toString());
// Assert results
// expect(file).toContain(JSON.stringify(buildParameter));
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
const environmentVariables = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
return {
name: x.EnvironmentVariable,
value: x.ParameterValue,
};
});
const combined = [...environmentVariables, ...secrets]
.filter((element) => element.value !== undefined && element.value !== '' && typeof element.value !== 'function')
.map((x) => {
if (typeof x.value === `string`) {
x.value = x.value.replace(/\s+/g, '');
}
return x;
})
.filter((element) => {
return !['UNITY_LICENSE', 'CUSTOM_JOB'].includes(element.name);
});
const newLinePurgedFile = file
.replace(/\s+/g, '')
.replace(new RegExp(`\\[${CloudRunnerStatics.logPrefix}\\]`, 'g'), '');
for (const element of combined) {
expect(newLinePurgedFile).toContain(`${element.name}`);
CloudRunnerLogger.log(`Contains ${element.name}`);
const fullNameEqualValue = `${element.name}=${element.value}`;
expect(newLinePurgedFile).toContain(fullNameEqualValue);
}
}, 1_000_000_000);
}
});

View File

@@ -0,0 +1,99 @@
import SharedWorkspaceLocking from '../services/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Simple Locking Flow`, async () => {
Cli.options.retainWorkspaces = true;
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
const isExpectedLockedAfterLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
expect(locks.length).toBe(0);
const isExpectedLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy();
}, 150000);
it.skip('All Locking Actions', async () => {
Cli.options.retainWorkspaces = true;
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
CloudRunnerLogger.log(
`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`IsWorkspaceLocked ${JSON.stringify(
await SharedWorkspaceLocking.IsWorkspaceLocked(`test-workspace-${uuidv4()}`, buildParameters),
)}`,
);
CloudRunnerLogger.log(
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4(), buildParameters),
)}`,
);
CloudRunnerLogger.log(
`CreateLockableWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`, buildParameters),
)}`,
);
CloudRunnerLogger.log(
`GetLockedWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${uuidv4()}`,
uuidv4(),
buildParameters,
),
)}`,
);
}, 3000000);
}
});