actions/unity-builder
2
0
Fork 0
mirror of https://github.com/game-ci/unity-builder.git synced 2026-02-08 19:39:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cloud Runner Develop - 1.0 R.C 1 (#437)

Browse Source 
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
2022-11-07 20:41:00 +00:00
committed by GitHub
parent 96555a0945
commit 4cca069ebb
76 changed files with 5325 additions and 2023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/model/cloud-runner/remote-client/caching.ts
View File

@@ -46,40 +46,49 @@ export class Caching {
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
const compressionSuffix = CloudRunner.buildParameters.useLz4Compression ? '.lz4' : '';
try {
if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
}
process.chdir(path.resolve(sourceFolder, '..'));
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
sourceFolder,
)}`,
);
}
// eslint-disable-next-line func-style
const formatFunction = function (format: string) {
const arguments_ = Array.prototype.slice.call(
[path.resolve(sourceFolder, '..'), cacheFolder, cacheArtifactName],
1,
);
const contents = await fs.promises.readdir(path.basename(sourceFolder));
CloudRunnerLogger.log(
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
);
return format.replace(/{(\d+)}/g, function (match, number) {
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
});
};
await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar ${path.basename(sourceFolder)}`);
assert(await fileExists(`${cacheArtifactName}.tar`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
if (CloudRunner.buildParameters.cachePushOverrideCommand) {
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand));
if (CloudRunner.buildParameters.cloudRunnerDebug) {
// await CloudRunnerSystem.Run(`tree -L 2 ./..`);
// await CloudRunnerSystem.Run(`tree -L 2`);
}
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`);
if (contents.length === 0) {
CloudRunnerLogger.log(
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
);
process.chdir(`${startPath}`);
return;
}
await CloudRunnerSystem.Run(
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path.basename(sourceFolder)}`,
);
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar`),
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
'cache archive exists inside cache folder',
);
} catch (error) {
@@ -90,6 +99,7 @@ export class Caching {
}
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
cacheArtifactName = cacheArtifactName.replace(' ', '');
const compressionSuffix = CloudRunner.buildParameters.useLz4Compression ? '.lz4' : '';
const startPath = process.cwd();
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
try {
@@ -101,38 +111,26 @@ export class Caching {
await fs.promises.mkdir(destinationFolder);
}
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`))
const latestInBranch = await (
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
)
.replace(/\n/g, ``)
.replace('.tar', '');
.replace(`.tar${compressionSuffix}`, '');
process.chdir(cacheFolder);
const cacheSelection =
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch;
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName
: latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
// eslint-disable-next-line func-style
const formatFunction = function (format: string) {
const arguments_ = Array.prototype.slice.call(
[path.resolve(destinationFolder, '..'), cacheFolder, cacheArtifactName],
1,
);
return format.replace(/{(\d+)}/g, function (match, number) {
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
});
};
if (CloudRunner.buildParameters.cachePullOverrideCommand) {
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand));
}
if (await fileExists(`${cacheSelection}.tar`)) {
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${fullResultsFolder}`);
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path.resolve(destinationFolder, '..');
@@ -152,15 +150,17 @@ export class Caching {
} else {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`);
RemoteClientLogger.logWarning(
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
}
}
} catch (error) {
process.chdir(`${startPath}`);
process.chdir(startPath);
throw error;
}
process.chdir(`${startPath}`);
process.chdir(startPath);
}
public static async handleCachePurging() {