mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-02-04 16:19:09 +08:00
Cloud runner develop - latest fixes (#524)
Cloud runner develop - latest fixes (#524)
This commit is contained in:
Frostebite
@@ -0,0 +1,86 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { BuildParameters, ImageTag } from '../../..';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import { Cli } from '../../../cli/cli';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from '../cloud-runner-suite.test';
|
||||
import * as fs from 'node:fs';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
|
||||
async function CreateParameters(overrides: any) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Caching', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
containerHookFiles: `debug-cache`,
|
||||
};
|
||||
if (CloudRunnerOptions.providerStrategy === `k8s`) {
|
||||
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
|
||||
}
|
||||
const buildParameter = await CreateParameters(overrides);
|
||||
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
|
||||
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const results = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
|
||||
const buildSucceededString = 'Build succeeded';
|
||||
|
||||
expect(results).toContain(libraryString);
|
||||
expect(results).toContain(buildSucceededString);
|
||||
expect(results).not.toContain(cachePushFail);
|
||||
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
|
||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||
await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`mkdir results`);
|
||||
await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
|
||||
await CloudRunnerSystem.Run(`tree -d ./results`);
|
||||
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
|
||||
expect(cacheFolderExists).toBeTruthy();
|
||||
}
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
|
||||
buildParameter2.cacheKey = buildParameter.cacheKey;
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
|
||||
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
|
||||
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
|
||||
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
|
||||
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for Library',
|
||||
);
|
||||
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for LFS',
|
||||
);
|
||||
|
||||
expect(build2ContainsCacheKey).toBeTruthy();
|
||||
expect(build2ContainsBuildSucceeded).toBeTruthy();
|
||||
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsNoLibraryMessage).toBeTruthy();
|
||||
}, 1_000_000_000);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { BuildParameters } from '../../..';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import { Cli } from '../../../cli/cli';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from '../cloud-runner-suite.test';
|
||||
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
|
||||
|
||||
async function CreateParameters(overrides: any) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Locking', () => {
|
||||
setups();
|
||||
it('Responds', () => {});
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
it(`Simple Locking End2End Flow`, async () => {
|
||||
const overrides: any = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
maxRetainedWorkspaces: 3,
|
||||
};
|
||||
const buildParameters = await CreateParameters(overrides);
|
||||
|
||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||
const runId = uuidv4();
|
||||
CloudRunner.buildParameters = buildParameters;
|
||||
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
|
||||
expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
|
||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||
const isExpectedUnlockedBeforeLocking =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
|
||||
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
|
||||
const result = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
expect(result).toBeTruthy();
|
||||
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
|
||||
expect(lines.map((x) => x.replace(`/`, ``)).includes(buildParameters.cacheKey));
|
||||
expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
|
||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||
const allLocks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
|
||||
expect(
|
||||
(
|
||||
await SharedWorkspaceLocking.ReadLines(
|
||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
|
||||
)
|
||||
).filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
(
|
||||
await SharedWorkspaceLocking.ReadLines(
|
||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
|
||||
)
|
||||
).filter((x) => x.endsWith(`${newWorkspaceName}_workspace`)),
|
||||
).toHaveLength(1);
|
||||
expect(allLocks.filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)).length).toBeGreaterThan(0);
|
||||
const isExpectedLockedAfterLocking =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
|
||||
expect(isExpectedLockedAfterLocking).toBeTruthy();
|
||||
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocksForWorkspace(
|
||||
newWorkspaceName,
|
||||
buildParameters,
|
||||
);
|
||||
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
|
||||
expect(locksBeforeRelease.length).toBe(1);
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
const locks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
|
||||
expect(locks.length).toBe(0);
|
||||
const isExpectedNotLockedAfterReleasing =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
|
||||
expect(isExpectedNotLockedAfterReleasing).toBeTruthy();
|
||||
const lockingResult2 = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
expect(lockingResult2).toBeTruthy();
|
||||
expect((await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true).toBeTruthy();
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
expect(
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false,
|
||||
).toBeTruthy();
|
||||
await SharedWorkspaceLocking.CleanupWorkspace(newWorkspaceName, buildParameters);
|
||||
CloudRunnerLogger.log(`Starting get or create`);
|
||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||
}, 350000);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { ImageTag } from '../../..';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from './../cloud-runner-suite.test';
|
||||
import * as fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
|
||||
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
|
||||
import { CreateParameters } from '../create-test-parameter';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
|
||||
describe('Cloud Runner Retain Workspace', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
maxRetainedWorkspaces: 1,
|
||||
};
|
||||
const buildParameter = await CreateParameters(overrides);
|
||||
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
|
||||
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const results = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
|
||||
const buildSucceededString = 'Build succeeded';
|
||||
|
||||
expect(results).toContain(libraryString);
|
||||
expect(results).toContain(buildSucceededString);
|
||||
expect(results).not.toContain(cachePushFail);
|
||||
|
||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
|
||||
expect(cacheFolderExists).toBeTruthy();
|
||||
await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache`);
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
|
||||
// await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache/${}`);
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
|
||||
buildParameter2.cacheKey = buildParameter.cacheKey;
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
|
||||
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
|
||||
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
|
||||
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
|
||||
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
|
||||
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
|
||||
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
|
||||
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for Library',
|
||||
);
|
||||
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for LFS',
|
||||
);
|
||||
|
||||
expect(build2ContainsCacheKey).toBeTruthy();
|
||||
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
|
||||
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
|
||||
expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy();
|
||||
expect(build2ContainsBuildSucceeded).toBeTruthy();
|
||||
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsNoLibraryMessage).toBeTruthy();
|
||||
}, 1_000_000_000);
|
||||
afterAll(async () => {
|
||||
await SharedWorkspaceLocking.CleanupWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters);
|
||||
if (
|
||||
fs.existsSync(`./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`)
|
||||
) {
|
||||
CloudRunnerLogger.log(
|
||||
`Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user