From 0d03f16d22f005b560653c3fd279d799665c0c8e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Fri, 5 Feb 2016 13:30:18 +0800 Subject: [PATCH] Add high scores bucket to bolt --- .gitignore | 1 + handlers/check.go | 81 ++++++++++++++++++++++++++++++++++- handlers/checks.go | 8 ---- handlers/high_scores.go | 88 ++++++++++++++++++++++++-------------- handlers/score_heap.go | 28 ++++++++++++ main.go | 4 ++ templates/high_scores.html | 8 ++-- 7 files changed, 172 insertions(+), 46 deletions(-) create mode 100644 handlers/score_heap.go diff --git a/.gitignore b/.gitignore index c27405f..587195d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ _testmain.go *.swp repos +goreportcard.db diff --git a/handlers/check.go b/handlers/check.go index 1826e50..48c0a98 100644 --- a/handlers/check.go +++ b/handlers/check.go @@ -1,6 +1,7 @@ package handlers import ( + "container/heap" "encoding/json" "fmt" "log" @@ -17,6 +18,10 @@ const ( // RepoBucket is the bucket in which repos will be cached in the bolt DB RepoBucket string = "repos" + + // HighScoreBucket is the bucket containing the names of the projects with the + // top 100 high scores + HighScoreBucket string = "high_scores" ) // CheckHandler handles the request for checking a repo @@ -63,7 +68,81 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { if b == nil { return fmt.Errorf("Repo bucket not found") } - return b.Put([]byte(repo), respBytes) + + // is this a new repo? if so, increase the count in the high scores bucket later + isNewRepo := b.Get([]byte(repo)) == nil + + err := b.Put([]byte(repo), respBytes) + if err != nil { + return err + } + + // check if we might need to update the high score list + if resp.Files < 100 { + // only repos with >= 100 files are considered for the high score list + return nil + } + + hsb := tx.Bucket([]byte(HighScoreBucket)) + if hsb == nil { + return fmt.Errorf("High score bucket not found") + } + // update total repos count + if isNewRepo { + totalInt := 0 + total := hsb.Get([]byte("total_repos")) + if total != nil { + err = json.Unmarshal(total, totalInt) + if err != nil { + return fmt.Errorf("Could not unmarshal total repos count: %v", err) + } + } + total, err = json.Marshal(totalInt + 1) + if err != nil { + return fmt.Errorf("Could not marshal total repos count: %v", err) + } + hsb.Put([]byte("total_repos"), total) + } + + scoreBytes := hsb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + scores := &scoreHeap{} + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 { + // lowest score on list is higher than this repo's score, so no need to add + return nil + } + // if this repo is already in the list, remove the original entry: + for i := range *scores { + if strings.Compare((*scores)[i].Repo, repo) == 0 { + heap.Remove(scores, i) + break + } + } + // now we can safely push it onto the heap + heap.Push(scores, scoreItem{ + Repo: repo, + Score: resp.Average * 100.0, + Files: resp.Files, + }) + if len(*scores) > 100 { + // trim heap if it's grown to over 100 + *scores = (*scores)[:100] + } + scoreBytes, err = json.Marshal(&scores) + if err != nil { + return err + } + err = hsb.Put([]byte("scores"), scoreBytes) + if err != nil { + return err + } + + return nil }) if err != nil { diff --git a/handlers/checks.go b/handlers/checks.go index f457e01..0bef255 100644 --- a/handlers/checks.go +++ b/handlers/checks.go @@ -1,7 +1,6 @@ package handlers import ( - "encoding/binary" "encoding/json" "errors" "fmt" @@ -49,13 +48,6 @@ func getFromCache(repo string) (checksResp, error) { return resp, nil } -// itob returns an 8-byte big endian representation of v. -func itob(v int) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(v)) - return b -} - type score struct { Name string `json:"name"` Description string `json:"description"` diff --git a/handlers/high_scores.go b/handlers/high_scores.go index 517bc35..94cd9dc 100644 --- a/handlers/high_scores.go +++ b/handlers/high_scores.go @@ -1,8 +1,16 @@ package handlers import ( + "container/heap" + "encoding/json" "fmt" + "log" "net/http" + "text/template" + "time" + + "github.com/boltdb/bolt" + "github.com/dustin/go-humanize" ) func add(x, y int) int { @@ -10,41 +18,55 @@ func add(x, y int) int { } func formatScore(x float64) string { - return fmt.Sprintf("%.2f", x*100) + return fmt.Sprintf("%.2f", x) } // HighScoresHandler handles the stats page func HighScoresHandler(w http.ResponseWriter, r *http.Request) { - // session, err := mgo.Dial(mongoURL) - // if err != nil { - // log.Println("ERROR: could not get collection:", err) - // http.Error(w, err.Error(), 500) - // return - // } - // defer session.Close() - // coll := session.DB(mongoDatabase).C(mongoCollection) - // - // var highScores []struct { - // Repo string - // Files int - // Average float64 - // } - // err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores) - // if err != nil { - // log.Println("ERROR: could not get high scores: ", err) - // http.Error(w, err.Error(), 500) - // return - // } - // - // count, err := coll.Count() - // if err != nil { - // log.Println("ERROR: could not get count of high scores: ", err) - // http.Error(w, err.Error(), 500) - // return - // } - // - // funcs := template.FuncMap{"add": add, "formatScore": formatScore} - // t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) - // - // t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))}) + // write to boltdb + db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + log.Println("Failed to open bolt database: ", err) + return + } + defer db.Close() + + count, scores := 0, &scoreHeap{} + err = db.View(func(tx *bolt.Tx) error { + hsb := tx.Bucket([]byte(HighScoreBucket)) + if hsb == nil { + return fmt.Errorf("High score bucket not found") + } + scoreBytes := hsb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + + total := hsb.Get([]byte("total_repos")) + if total == nil { + count = 0 + } else { + json.Unmarshal(total, &count) + } + return nil + }) + + if err != nil { + log.Println("ERROR: Failed to load high scores from bolt database: ", err) + http.Error(w, err.Error(), 500) + return + } + + funcs := template.FuncMap{"add": add, "formatScore": formatScore} + t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) + + sortedScores := make([]scoreItem, len(*scores)) + for i := range sortedScores { + sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem) + } + + t.Execute(w, map[string]interface{}{"HighScores": sortedScores, "Count": humanize.Comma(int64(count))}) } diff --git a/handlers/score_heap.go b/handlers/score_heap.go new file mode 100644 index 0000000..873aa57 --- /dev/null +++ b/handlers/score_heap.go @@ -0,0 +1,28 @@ +package handlers + +type scoreItem struct { + Repo string `json:"repo"` + Score float64 `json:"score"` + Files int `json:"files"` +} + +// An scoreHeap is a min-heap of ints. +type scoreHeap []scoreItem + +func (h scoreHeap) Len() int { return len(h) } +func (h scoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score } +func (h scoreHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *scoreHeap) Push(x interface{}) { + // Push and Pop use pointer receivers because they modify the slice's length, + // not just its contents. + *h = append(*h, x.(scoreItem)) +} + +func (h *scoreHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} diff --git a/main.go b/main.go index b3e9478..0b0ff04 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,10 @@ func initDB() error { err = db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(handlers.RepoBucket)) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists([]byte(handlers.HighScoreBucket)) return err }) return err diff --git a/templates/high_scores.html b/templates/high_scores.html index 2bb23c1..2f3ea5c 100644 --- a/templates/high_scores.html +++ b/templates/high_scores.html @@ -12,10 +12,10 @@ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - + ga('create', 'UA-58936835-1', 'auto'); ga('send', 'pageview'); - +