This commit is contained in:
frostebite
2026-01-23 21:13:39 +00:00
parent b2cb6ebb19
commit 1cf4f0326b
8 changed files with 107 additions and 60 deletions

View File

@@ -19,6 +19,7 @@ import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
import CloudRunnerResult from './services/core/cloud-runner-result';
import CloudRunnerOptions from './options/cloud-runner-options';
import ResourceTracking from './services/core/resource-tracking';
class CloudRunner {
public static Provider: ProviderInterface;
@@ -37,6 +38,8 @@ class CloudRunner {
CloudRunnerLogger.setup();
CloudRunnerLogger.log(`Setting up cloud runner`);
CloudRunner.buildParameters = buildParameters;
ResourceTracking.logAllocationSummary('setup');
await ResourceTracking.logDiskUsageSnapshot('setup');
if (CloudRunner.buildParameters.githubCheckId === ``) {
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
}

View File

@@ -287,6 +287,10 @@ class CloudRunnerOptions {
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
}
public static get resourceTracking(): boolean {
return CloudRunnerOptions.getInput('resourceTracking') === 'true';
}
public static get useLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
}

View File

@@ -15,6 +15,7 @@ import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { AwsClientFactory } from './aws-client-factory';
import ResourceTracking from '../../services/core/resource-tracking';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
@@ -90,6 +91,8 @@ class AWSBuildEnvironment implements ProviderInterface {
secrets: CloudRunnerSecret[],
): Promise<string> {
process.env.AWS_REGION = Input.region;
ResourceTracking.logAllocationSummary('aws workflow');
await ResourceTracking.logDiskUsageSnapshot('aws workflow (host)');
AwsClientFactory.getECS();
const CF = AwsClientFactory.getCloudFormation();
AwsClientFactory.getKinesis();

View File

@@ -17,6 +17,7 @@ import { ProviderWorkflow } from '../provider-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import { KubernetesRole } from './kubernetes-role';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import ResourceTracking from '../../services/core/resource-tracking';
class Kubernetes implements ProviderInterface {
public static Instance: Kubernetes;
@@ -137,6 +138,9 @@ class Kubernetes implements ProviderInterface {
): Promise<string> {
try {
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
ResourceTracking.logAllocationSummary('k8s workflow');
await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)');
await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)');
// Setup
const id =

View File

@@ -14,11 +14,13 @@ import GitHub from '../../github';
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../options/cloud-runner-options';
import ResourceTracking from '../services/core/resource-tracking';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() {
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
@@ -206,6 +208,7 @@ export class RemoteClient {
// that read from the log file rather than stdout
RemoteClientLogger.log(successMessage);
CloudRunnerLogger.log(successMessage);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-post-build (end)');
return new Promise((result) => result(``));
}

View File

@@ -0,0 +1,84 @@
import CloudRunnerLogger from './cloud-runner-logger';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerSystem } from './cloud-runner-system';
class ResourceTracking {
static isEnabled(): boolean {
return (
CloudRunnerOptions.resourceTracking ||
CloudRunnerOptions.cloudRunnerDebug ||
process.env['cloudRunnerTests'] === 'true'
);
}
static logAllocationSummary(context: string) {
if (!ResourceTracking.isEnabled()) {
return;
}
const buildParameters = CloudRunner.buildParameters;
const allocations = {
providerStrategy: buildParameters.providerStrategy,
containerCpu: buildParameters.containerCpu,
containerMemory: buildParameters.containerMemory,
dockerCpuLimit: buildParameters.dockerCpuLimit,
dockerMemoryLimit: buildParameters.dockerMemoryLimit,
kubeVolumeSize: buildParameters.kubeVolumeSize,
kubeStorageClass: buildParameters.kubeStorageClass,
kubeVolume: buildParameters.kubeVolume,
containerNamespace: buildParameters.containerNamespace,
storageProvider: buildParameters.storageProvider,
rcloneRemote: buildParameters.rcloneRemote,
dockerWorkspacePath: buildParameters.dockerWorkspacePath,
cacheKey: buildParameters.cacheKey,
maxRetainedWorkspaces: buildParameters.maxRetainedWorkspaces,
useCompressionStrategy: buildParameters.useCompressionStrategy,
useLargePackages: buildParameters.useLargePackages,
ephemeralStorageRequest: process.env['cloudRunnerTests'] === 'true' ? 'not set' : '2Gi',
};
CloudRunnerLogger.log(`[ResourceTracking] Allocation summary (${context}):`);
CloudRunnerLogger.log(JSON.stringify(allocations, undefined, 2));
}
static async logDiskUsageSnapshot(context: string) {
if (!ResourceTracking.isEnabled()) {
return;
}
CloudRunnerLogger.log(`[ResourceTracking] Disk usage snapshot (${context})`);
await ResourceTracking.runAndLog('df -h', 'df -h');
await ResourceTracking.runAndLog('du -sh .', 'du -sh .');
await ResourceTracking.runAndLog('du -sh ./cloud-runner-cache', 'du -sh ./cloud-runner-cache');
await ResourceTracking.runAndLog('du -sh ./temp', 'du -sh ./temp');
await ResourceTracking.runAndLog('du -sh ./logs', 'du -sh ./logs');
}
static async logK3dNodeDiskUsage(context: string) {
if (!ResourceTracking.isEnabled()) {
return;
}
const nodes = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
CloudRunnerLogger.log(`[ResourceTracking] K3d node disk usage (${context})`);
for (const node of nodes) {
await ResourceTracking.runAndLog(
`k3d node ${node}`,
`docker exec ${node} sh -c "df -h /var/lib/rancher/k3s 2>/dev/null || df -h / 2>/dev/null || true" || true`,
);
}
}
private static async runAndLog(label: string, command: string) {
try {
const output = await CloudRunnerSystem.Run(command, true, true);
const trimmed = output.trim();
CloudRunnerLogger.log(`[ResourceTracking] ${label}:\n${trimmed || 'no output'}`);
} catch (error: any) {
CloudRunnerLogger.log(`[ResourceTracking] ${label} failed: ${error?.message || error}`);
}
}
}
export default ResourceTracking;