actions/unity-builder
2
0
Fork 0
mirror of https://github.com/game-ci/unity-builder.git synced 2026-02-05 17:29:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(aws): increase backoff and handle throttling in DescribeTasks/GetRecords

Browse Source
This commit is contained in:
Frostebite
2025-09-06 02:13:31 +01:00
parent f7725a72d6
commit af988e6d2a
3 changed files with 32 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@@ -132,8 +132,8 @@ class AWSTaskRunner {
}
static async describeTasks(clusterName: string, taskArn: string) {
const maxAttempts = 6;
let delayMs = 500;
const maxAttempts = 10;
let delayMs = 1000;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const tasks = await AWSTaskRunner.ECS.send(
@@ -148,6 +148,7 @@ class AWSTaskRunner {
if (!isThrottle || attempt === maxAttempts) {
throw error;
}
CloudRunnerLogger.log(`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${delayMs}ms`);
await new Promise((r) => setTimeout(r, delayMs));
delayMs *= 2;
}
@@ -188,7 +189,18 @@ class AWSTaskRunner {
output: string,
shouldCleanup: boolean,
) {
const records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
let records: any;
try {
records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
} catch (error: any) {
const isThrottle = error?.name === 'ThrottlingException' || /rate exceeded/i.test(String(error?.message));
if (isThrottle) {
CloudRunnerLogger.log(`AWS throttled GetRecords, backing off 1000ms`);
await new Promise((r) => setTimeout(r, 1000));
return { iterator, shouldReadLogs, output, shouldCleanup };
}
throw error;
}
iterator = records.NextShardIterator || '';
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
records,