From efb34e0c5b017136c0efa0d91965684e207914f5 Mon Sep 17 00:00:00 2001 From: yaodeshun Date: Thu, 21 Aug 2025 15:52:58 +0800 Subject: [PATCH 1/2] =?UTF-8?q?sdk=E5=92=8C=E6=B5=8B=E8=AF=95=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package/gamelog/sdk.go | 356 +++++++++++++++++++++++++++ package/gamelog/test/gamelog_test.go | 51 ++++ 2 files changed, 407 insertions(+) create mode 100644 package/gamelog/sdk.go create mode 100644 package/gamelog/test/gamelog_test.go diff --git a/package/gamelog/sdk.go b/package/gamelog/sdk.go new file mode 100644 index 0000000..0f0443a --- /dev/null +++ b/package/gamelog/sdk.go @@ -0,0 +1,356 @@ +package gamelog + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" + "github.com/gogf/gf/v2/os/gfile" + "github.com/gogf/gf/v2/os/gtime" +) + +type sendBody struct { + Pid string `json:"pid"` + Data [][]any `json:"data"` +} + +// todo 游戏日志对象 +type GameLog struct { + Uid string // 唯一uid + Event string // 事件名 + Property map[string]any // 事件属性 + EventTimems int64 // 时间戳毫秒级别 + EventTimeLoc string // 带时区的本地时间字符串 +} + +type SDKConfig struct { + // 配置变量 + Pid string // 项目id + BaseUrl string // 日志服务器地址 + ReportSk string // 上报解密key + FlushInterval int // 刷新间隔 + DiskBakPath string // 磁盘备份路径 + RetryN int // 每N次重试 + + reportN int +} + +type SDK struct { + // 控制变量 + wg sync.WaitGroup + shutdown chan struct{} + mu sync.Mutex + sdkConfig *SDKConfig + bufferChan chan GameLog // 日志队列 + buffer []GameLog // 日志队列 +} + +var ( + ctx = context.Background() + gamelogClient *gclient.Client + + // location map + locationMap map[string]*time.Location = map[string]*time.Location{} +) + +func (sdk *SDK) varinit() error { + sdk.sdkConfig = &SDKConfig{} + + _pid, err := g.Config().Get(ctx, "angergs.bisdk.pid") + if err != nil { + return err + } + sdk.sdkConfig.Pid = _pid.String() + + _baseUrl, err := g.Config().Get(ctx, "angergs.bisdk.recodeServerBaseUrl") + if err != nil { + return err + } + sdk.sdkConfig.BaseUrl = _baseUrl.String() + + _sk, err := g.Config().Get(ctx, "angergs.bisdk.reportSk") + if err != nil { + return err + } + sdk.sdkConfig.ReportSk = _sk.String() + + _flushInterval, err := g.Config().Get(ctx, "angergs.bisdk.flushInterval") + if err != nil { + return err + } + sdk.sdkConfig.FlushInterval = _flushInterval.Int() + + _diskBakPath, err := g.Config().Get(ctx, "angergs.bisdk.diskBakPath") + if err != nil { + return err + } + sdk.sdkConfig.DiskBakPath = _diskBakPath.String() + + _retryN, err := g.Config().Get(ctx, "angergs.bisdk.retryN") + if err != nil { + return err + } + sdk.sdkConfig.RetryN = _retryN.Int() + + g.Log().Infof(ctx, "[GameLog]client init success, config: %v", sdk.sdkConfig) + return nil +} + +func (sdk *SDK) checkConfig() error { + config := sdk.sdkConfig + if config.Pid == "" { + return fmt.Errorf("pid is empty") + } + if config.BaseUrl == "" { + return fmt.Errorf("baseUrl is empty") + } + if config.ReportSk == "" { + return fmt.Errorf("reportSk is empty") + } + if config.FlushInterval <= 0 { + return fmt.Errorf("flushInterval is invalid") + } + if config.DiskBakPath == "" { + return fmt.Errorf("diskBakPath is empty") + } + if config.RetryN == 0 { + config.RetryN = 10 + } + config.DiskBakPath = strings.TrimSuffix(config.DiskBakPath, "/") + + return nil +} + +func INIT(config *SDKConfig) (*SDK, error) { + // 加载并检查配置 + sdk := &SDK{} + if config != nil { + sdk.sdkConfig = config + } else if err := sdk.varinit(); err != nil { // 可以读goframe的配置 + return nil, err + } + if err := sdk.checkConfig(); err != nil { + return nil, err + } + gamelogClient = g.Client() + + // 初始化队列 + sdk.shutdown = make(chan struct{}) + sdk.bufferChan = make(chan GameLog, 1000) + sdk.buffer = make([]GameLog, 0, 100) + // 加载失败日志 + failLogs, err := sdk.loadFailLogs4disk() + if err != nil && len(failLogs) > 0 { + sdk.buffer = append(sdk.buffer, failLogs...) + } + + // 开启协程进行日志发送 + sdk.wg = sync.WaitGroup{} + sdk.wg.Add(1) + go func() { + defer sdk.wg.Done() + ticker := time.NewTicker(time.Duration(sdk.sdkConfig.FlushInterval) * time.Second) + defer ticker.Stop() + + for { + select { + case <-sdk.shutdown: + // 关闭时, 上传一次并备份失败数据 + g.Log().Infof(ctx, "[GameLog]begin shutdown and flush last") + sdk.flush() + return + case log := <-sdk.bufferChan: + sdk.buffer = append(sdk.buffer, log) + case <-ticker.C: + sdk.flush() + + } + } + }() + return sdk, nil +} + +// 从磁盘加载失败日志 +func (sdk *SDK) loadFailLogs4disk() (logs []GameLog, err error) { + if !gfile.Exists(sdk.sdkConfig.DiskBakPath) { + return + } + // 遍历diskBakPath下所有failBufferxxx.bak.log文件， 读取到log中 + files, err := gfile.ScanDir(sdk.sdkConfig.DiskBakPath, "failBuffer*.bak.log") + logs = []GameLog{} + if err != nil { + return + } + // 读取每个备份文件 + for _, fp := range files { + // 每一行都是一次失败的记录 + gfile.ReadLines(fp, func(line string) error { + _logs := []GameLog{} + err := json.Unmarshal([]byte(line), &_logs) + if err != nil { + return err + } + // 合并到总日志列表 + logs = append(logs, _logs...) + return nil + }) + g.Log().Infof(ctx, "[GameLog]load %d faillogs from %s", len(logs), fp) + gfile.Remove(fp) + } + return + +} + +// 备份失败日志追加到磁盘 +func (sdk *SDK) bakFailLogs2disk(failLogs []GameLog) { + bakPath := fmt.Sprintf("%s/failBuffer%s.bak.log", sdk.sdkConfig.DiskBakPath, gtime.Now().Format("YmdH")) + content, _ := json.Marshal(failLogs) + gfile.PutContentsAppend(bakPath, string(content)+"\n") + g.Log().Infof(ctx, "[GameLog]backup fail buffer to %s", bakPath) +} + +// 优雅关闭 +func (sdk *SDK) Shutdown() { + close(sdk.shutdown) + sdk.wg.Wait() +} + +// 日志时间格式 +const datetimeFmt = time.DateOnly + " " + time.TimeOnly + +// 记录日志 +func (sdk *SDK) Log(uid, event string, property map[string]any, timezone string) { + loc := time.Local + if _, ok := locationMap[timezone]; !ok { + location, err := time.LoadLocation(timezone) + if err != nil { + g.Log().Warningf(ctx, "[GameLog]load location error, try use local timezone: %v", err) + } else { + locationMap[timezone] = location + loc = location + } + } + log := GameLog{ + Uid: uid, + Event: event, + Property: property, + EventTimems: gtime.Now().TimestampMilli(), + EventTimeLoc: gtime.Now().In(loc).Format(datetimeFmt), + } + // 线程安全 + sdk.bufferChan <- log +} + +// 按服务器时区记录日志 +func (sdk *SDK) LogLtz(uid, event string, property map[string]any) { + sdk.Log(uid, event, property, time.Local.String()) +} + +// 这个方法只会在内部协程调用 +func (sdk *SDK) flush() { + sdk.mu.Lock() + defer sdk.mu.Unlock() + if len(sdk.buffer) == 0 { + return + } + + batch := make([]GameLog, len(sdk.buffer)) + copy(batch, sdk.buffer) + sdk.buffer = sdk.buffer[:0] + + // 第N次的时候加载失败数据进行尝试 + if sdk.sdkConfig.reportN != 0 && sdk.sdkConfig.reportN%sdk.sdkConfig.RetryN == 0 { + faillogs, err := sdk.loadFailLogs4disk() + if err != nil { + g.Log().Errorf(ctx, "[GameLog]load fail logs error: %v", err) + } + // 如果有失败日志则加入到批量数组中 + if len(faillogs) > 0 { + batch = append(batch, faillogs...) + } + } + sdk.send(batch) +} + +// 发送消息 +func (sdk *SDK) send(logs []GameLog) { + waitSecond := time.Duration(sdk.sdkConfig.FlushInterval/4) * time.Second + timeoutCtx, cancel := context.WithTimeout(context.Background(), waitSecond) + defer cancel() + data := make([][]any, 0, len(logs)) + // logs 拆分成二维数组 + for _, log := range logs { + propertyJson, _ := json.Marshal(log.Property) + data = append(data, []any{ + log.Uid, + log.Event, + string(propertyJson), + log.EventTimems, + log.EventTimeLoc, + }) + } + // json化 + sbody := sendBody{ + Pid: sdk.sdkConfig.Pid, + Data: data, + } + jsonBody, _ := json.Marshal(sbody) + + // giz压缩 + gzBody := bytes.NewBuffer([]byte{}) + gz := gzip.NewWriter(gzBody) + gz.Write(jsonBody) + gz.Close() + + // XOR 加密 + xorBody := bytesXOR(gzBody.Bytes(), []byte(sdk.sdkConfig.ReportSk)) + + sdk.sdkConfig.reportN += 1 + res, err := gamelogClient.Post(timeoutCtx, sdk.sdkConfig.BaseUrl+"/report/event", xorBody) + defer func() { + cerr := res.Close() + if cerr != nil { + g.Log().Errorf(ctx, "[GameLog]close response error: %v", cerr) + } + }() + // 失败重新加入缓冲区 + if err != nil { + sdk.bakFailLogs2disk(logs) + g.Log().Warningf(ctx, "[GameLog]send log error, bak to fail buffer(%d): %v", len(logs), err) + return + } + httpcode := res.StatusCode + resBody := res.ReadAllString() + // 收集器拦截, 重新加入缓冲区 + if httpcode != http.StatusOK { + sdk.bakFailLogs2disk(logs) + g.Log().Warningf(ctx, "[GameLog]send log error, bak to fail buffer(%d): %v", len(logs), resBody) + } +} + +// 混淆 +func bytesXOR(data []byte, key []byte) []byte { + obfuscated := make([]byte, len(data)) + keyLen := len(key) + if keyLen == 0 { + return data + } + + for i := range data { + obfuscated[i] = data[i] ^ key[i%keyLen] + } + return obfuscated + + // // 使用示例 + // key := []byte{0x12, 0x34, 0x56, 0x78} + // obfuscated := multiXorObfuscate(original, key) + // deobfuscated := multiXorObfuscate(obfuscated, key) // 解密 +} diff --git a/package/gamelog/test/gamelog_test.go b/package/gamelog/test/gamelog_test.go new file mode 100644 index 0000000..252b919 --- /dev/null +++ b/package/gamelog/test/gamelog_test.go @@ -0,0 +1,51 @@ +package test + +import ( + "strings" + "testing" + "time" + + "github.com/ayflying/utility_go/package/gamelog" + "github.com/gogf/gf/v2/test/gtest" + "github.com/gogf/gf/v2/util/grand" + "github.com/google/uuid" +) + +func TestGamelog(t *testing.T) { + glsdk, err := gamelog.INIT(&gamelog.SDKConfig{ + // 必填 + Pid: "test5", // 项目ID + BaseUrl: "http://47.76.178.47:10101", // 香港测试服上报地址 + // BaseUrl: "http://127.0.0.1:10101", // 本次测试上报地址 + ReportSk: "sngame2025", // xor混淆key + FlushInterval: 5, // 上报间隔 + DiskBakPath: "gamelog", // 本地磁盘备份, 用于意外情况下临时保存日志, 请确保该目录持久化(容器内要挂载). 每次启动时或每N次上报时加载到失败队列 + // 可填 + RetryN: 2, // 默认每10次上传检查一次磁盘的失败数据 + }) + + // 随机测试事件和属性 + events := []string{"e1", "e2", "e3", "e4"} + pms := []map[string]any{ + {"a": "1"}, + {"a": "2"}, + {"a": "3"}, + {"a": "4"}, + } + if err != nil { + t.Fatal(err) + } + gtest.C(t, func(t *gtest.T) { + go func() { + for { + uuidval, _ := uuid.NewUUID() + randUid := strings.ReplaceAll(uuidval.String(), "-", "") + glsdk.LogLtz(randUid, events[grand.Intn(len(events))], pms[grand.Intn(len(pms))]) + time.Sleep(time.Millisecond * 100) + } + }() + time.Sleep(time.Second * 14) + // 模拟等待信号后优雅关闭 + glsdk.Shutdown() + }) +} From d0cad61028071edef78c5b8889c1450b3deaa3ca Mon Sep 17 00:00:00 2001 From: yaodeshun Date: Thu, 21 Aug 2025 16:09:10 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=B8=80=E4=BA=9Berr=E6=8E=A5=E5=8F=97?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package/gamelog/sdk.go | 44 +++++++++++++++++++++------- package/gamelog/test/gamelog_test.go | 3 +- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/package/gamelog/sdk.go b/package/gamelog/sdk.go index 0f0443a..91fa42b 100644 --- a/package/gamelog/sdk.go +++ b/package/gamelog/sdk.go @@ -39,6 +39,7 @@ type SDKConfig struct { FlushInterval int // 刷新间隔 DiskBakPath string // 磁盘备份路径 RetryN int // 每N次重试 + ChanSize int // 信道大小, 默认1000 reportN int } @@ -100,6 +101,12 @@ func (sdk *SDK) varinit() error { } sdk.sdkConfig.RetryN = _retryN.Int() + _chanSize, err := g.Config().Get(ctx, "angergs.bisdk.chanSize") + if err != nil { + return err + } + sdk.sdkConfig.ChanSize = _chanSize.Int() + g.Log().Infof(ctx, "[GameLog]client init success, config: %v", sdk.sdkConfig) return nil } @@ -124,6 +131,9 @@ func (sdk *SDK) checkConfig() error { if config.RetryN == 0 { config.RetryN = 10 } + if config.ChanSize == 0 { + config.ChanSize = 1000 + } config.DiskBakPath = strings.TrimSuffix(config.DiskBakPath, "/") return nil @@ -148,7 +158,9 @@ func INIT(config *SDKConfig) (*SDK, error) { sdk.buffer = make([]GameLog, 0, 100) // 加载失败日志 failLogs, err := sdk.loadFailLogs4disk() - if err != nil && len(failLogs) > 0 { + if err != nil { + g.Log().Errorf(ctx, "[GameLog]load fail logs error: %v", err) + } else if len(failLogs) > 0 { sdk.buffer = append(sdk.buffer, failLogs...) } @@ -212,7 +224,11 @@ func (sdk *SDK) loadFailLogs4disk() (logs []GameLog, err error) { // 备份失败日志追加到磁盘 func (sdk *SDK) bakFailLogs2disk(failLogs []GameLog) { bakPath := fmt.Sprintf("%s/failBuffer%s.bak.log", sdk.sdkConfig.DiskBakPath, gtime.Now().Format("YmdH")) - content, _ := json.Marshal(failLogs) + content, err := json.Marshal(failLogs) + if err != nil { + g.Log().Errorf(ctx, "[GameLog]marshal fail logs error: %v", err) + return + } gfile.PutContentsAppend(bakPath, string(content)+"\n") g.Log().Infof(ctx, "[GameLog]backup fail buffer to %s", bakPath) } @@ -288,7 +304,11 @@ func (sdk *SDK) send(logs []GameLog) { data := make([][]any, 0, len(logs)) // logs 拆分成二维数组 for _, log := range logs { - propertyJson, _ := json.Marshal(log.Property) + propertyJson, err := json.Marshal(log.Property) + if err != nil { + g.Log().Errorf(ctx, "[GameLog]skip log parse, marshal property error: %v", err) + continue + } data = append(data, []any{ log.Uid, log.Event, @@ -302,7 +322,11 @@ func (sdk *SDK) send(logs []GameLog) { Pid: sdk.sdkConfig.Pid, Data: data, } - jsonBody, _ := json.Marshal(sbody) + jsonBody, err := json.Marshal(sbody) + if err != nil { + g.Log().Errorf(ctx, "[GameLog]marshal send body error: %v", err) + return + } // giz压缩 gzBody := bytes.NewBuffer([]byte{}) @@ -315,18 +339,18 @@ func (sdk *SDK) send(logs []GameLog) { sdk.sdkConfig.reportN += 1 res, err := gamelogClient.Post(timeoutCtx, sdk.sdkConfig.BaseUrl+"/report/event", xorBody) - defer func() { - cerr := res.Close() - if cerr != nil { - g.Log().Errorf(ctx, "[GameLog]close response error: %v", cerr) - } - }() // 失败重新加入缓冲区 if err != nil { sdk.bakFailLogs2disk(logs) g.Log().Warningf(ctx, "[GameLog]send log error, bak to fail buffer(%d): %v", len(logs), err) return } + defer func() { + cerr := res.Close() + if cerr != nil { + g.Log().Errorf(ctx, "[GameLog]close response error: %v", cerr) + } + }() httpcode := res.StatusCode resBody := res.ReadAllString() // 收集器拦截, 重新加入缓冲区 diff --git a/package/gamelog/test/gamelog_test.go b/package/gamelog/test/gamelog_test.go index 252b919..e1cbc89 100644 --- a/package/gamelog/test/gamelog_test.go +++ b/package/gamelog/test/gamelog_test.go @@ -21,7 +21,8 @@ func TestGamelog(t *testing.T) { FlushInterval: 5, // 上报间隔 DiskBakPath: "gamelog", // 本地磁盘备份, 用于意外情况下临时保存日志, 请确保该目录持久化(容器内要挂载). 每次启动时或每N次上报时加载到失败队列 // 可填 - RetryN: 2, // 默认每10次上传检查一次磁盘的失败数据 + RetryN: 2, // 默认每10次, 上传检查一次磁盘的失败数据 + ChanSize: 500, // 默认1000, 信道size }) // 随机测试事件和属性