From ecd6fda13462e64f614dd60d6614ceb59412b3aa Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 4 Feb 2016 20:43:57 +0800 Subject: [PATCH 1/9] First pass of writing to and loading from bolt --- handlers/check.go | 43 +++++++++++++++++-------- handlers/checks.go | 50 +++++++++++++++++++---------- handlers/high_scores.go | 70 +++++++++++++++++++---------------------- main.go | 23 ++++++++++++++ 4 files changed, 119 insertions(+), 67 deletions(-) diff --git a/handlers/check.go b/handlers/check.go index d4409c4..1826e50 100644 --- a/handlers/check.go +++ b/handlers/check.go @@ -2,12 +2,21 @@ package handlers import ( "encoding/json" + "fmt" "log" "net/http" "strings" + "time" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "github.com/boltdb/bolt" +) + +const ( + // DBPath is the relative (or absolute) path to the bolt database file + DBPath string = "goreportcard.db" + + // RepoBucket is the bucket in which repos will be cached in the bolt DB + RepoBucket string = "repos" ) // CheckHandler handles the request for checking a repo @@ -31,26 +40,34 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { return } - b, err := json.Marshal(resp) + respBytes, err := json.Marshal(resp) if err != nil { log.Println("ERROR: could not marshal json:", err) http.Error(w, err.Error(), 500) return } - w.Write(b) + w.Write(respBytes) - // write to mongo - session, err := mgo.Dial(mongoURL) + // write to boltdb + db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { - log.Println("Failed to get mongo collection: ", err) + log.Println("Failed to open bolt database: ", err) return } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) - log.Printf("Upserting repo %s...", repo) - _, err = coll.Upsert(bson.M{"repo": repo}, resp) + defer db.Close() + + log.Printf("Saving repo %q to cache...", repo) + + err = db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(RepoBucket)) + if b == nil { + return fmt.Errorf("Repo bucket not found") + } + return b.Put([]byte(repo), respBytes) + }) + if err != nil { - log.Println("Mongo writing error:", err) - return + log.Println("Bolt writing error:", err) } + return } diff --git a/handlers/checks.go b/handlers/checks.go index bf19dc9..f457e01 100644 --- a/handlers/checks.go +++ b/handlers/checks.go @@ -1,6 +1,9 @@ package handlers import ( + "encoding/binary" + "encoding/json" + "errors" "fmt" "log" "os" @@ -8,36 +11,51 @@ import ( "strings" "time" + "github.com/boltdb/bolt" "github.com/gojp/goreportcard/check" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -var ( - mongoURL = "mongodb://127.0.0.1:27017" - mongoDatabase = "goreportcard" - mongoCollection = "reports" ) func getFromCache(repo string) (checksResp, error) { - // try and fetch from mongo - session, err := mgo.Dial(mongoURL) + // try and fetch from boltdb + db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { - return checksResp{}, fmt.Errorf("Failed to get mongo collection during GET: %v", err) + return checksResp{}, fmt.Errorf("Failed to open bolt database during GET: %v", err) } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) + defer db.Close() + resp := checksResp{} - err = coll.Find(bson.M{"repo": repo}).One(&resp) + err = db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(RepoBucket)) + if b == nil { + return errors.New("No repo bucket") + } + cached := b.Get([]byte(repo)) + if cached == nil { + return fmt.Errorf("%q not found in cache", repo) + } + + err = json.Unmarshal(cached, &resp) + if err != nil { + return fmt.Errorf("failed to parse JSON for %q in cache", repo) + } + return nil + }) + if err != nil { - return checksResp{}, fmt.Errorf("Failed to fetch %q from mongo: %v", repo, err) + return resp, err } resp.LastRefresh = resp.LastRefresh.UTC() - return resp, nil } +// itob returns an 8-byte big endian representation of v. +func itob(v int) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(v)) + return b +} + type score struct { Name string `json:"name"` Description string `json:"description"` diff --git a/handlers/high_scores.go b/handlers/high_scores.go index 7021188..517bc35 100644 --- a/handlers/high_scores.go +++ b/handlers/high_scores.go @@ -2,13 +2,7 @@ package handlers import ( "fmt" - "log" "net/http" - "text/template" - - "github.com/dustin/go-humanize" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" ) func add(x, y int) int { @@ -21,36 +15,36 @@ func formatScore(x float64) string { // HighScoresHandler handles the stats page func HighScoresHandler(w http.ResponseWriter, r *http.Request) { - session, err := mgo.Dial(mongoURL) - if err != nil { - log.Println("ERROR: could not get collection:", err) - http.Error(w, err.Error(), 500) - return - } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) - - var highScores []struct { - Repo string - Files int - Average float64 - } - err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores) - if err != nil { - log.Println("ERROR: could not get high scores: ", err) - http.Error(w, err.Error(), 500) - return - } - - count, err := coll.Count() - if err != nil { - log.Println("ERROR: could not get count of high scores: ", err) - http.Error(w, err.Error(), 500) - return - } - - funcs := template.FuncMap{"add": add, "formatScore": formatScore} - t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) - - t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))}) + // session, err := mgo.Dial(mongoURL) + // if err != nil { + // log.Println("ERROR: could not get collection:", err) + // http.Error(w, err.Error(), 500) + // return + // } + // defer session.Close() + // coll := session.DB(mongoDatabase).C(mongoCollection) + // + // var highScores []struct { + // Repo string + // Files int + // Average float64 + // } + // err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores) + // if err != nil { + // log.Println("ERROR: could not get high scores: ", err) + // http.Error(w, err.Error(), 500) + // return + // } + // + // count, err := coll.Count() + // if err != nil { + // log.Println("ERROR: could not get count of high scores: ", err) + // http.Error(w, err.Error(), 500) + // return + // } + // + // funcs := template.FuncMap{"add": add, "formatScore": formatScore} + // t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) + // + // t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))}) } diff --git a/main.go b/main.go index 01ea93b..b3e9478 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,9 @@ import ( "net/http" "os" "regexp" + "time" + "github.com/boltdb/bolt" "github.com/gojp/goreportcard/handlers" ) @@ -31,11 +33,32 @@ func makeHandler(name string, fn func(http.ResponseWriter, *http.Request, string } } +// initDB opens the bolt database file (or creates it if it does not exist), and creates +// a bucket for saving the repos, also only if it does not exist. +func initDB() error { + db, err := bolt.Open(handlers.DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return err + } + defer db.Close() + + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(handlers.RepoBucket)) + return err + }) + return err +} + func main() { if err := os.MkdirAll("repos/src/github.com", 0755); err != nil && !os.IsExist(err) { log.Fatal("ERROR: could not create repos dir: ", err) } + // initialize database + if err := initDB(); err != nil { + log.Fatal("ERROR: could not open bolt db: ", err) + } + http.HandleFunc("/assets/", handlers.AssetsHandler) http.HandleFunc("/checks", handlers.CheckHandler) http.HandleFunc("/report/", makeHandler("report", handlers.ReportHandler)) From 0d03f16d22f005b560653c3fd279d799665c0c8e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Fri, 5 Feb 2016 13:30:18 +0800 Subject: [PATCH 2/9] Add high scores bucket to bolt --- .gitignore | 1 + handlers/check.go | 81 ++++++++++++++++++++++++++++++++++- handlers/checks.go | 8 ---- handlers/high_scores.go | 88 ++++++++++++++++++++++++-------------- handlers/score_heap.go | 28 ++++++++++++ main.go | 4 ++ templates/high_scores.html | 8 ++-- 7 files changed, 172 insertions(+), 46 deletions(-) create mode 100644 handlers/score_heap.go diff --git a/.gitignore b/.gitignore index c27405f..587195d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ _testmain.go *.swp repos +goreportcard.db diff --git a/handlers/check.go b/handlers/check.go index 1826e50..48c0a98 100644 --- a/handlers/check.go +++ b/handlers/check.go @@ -1,6 +1,7 @@ package handlers import ( + "container/heap" "encoding/json" "fmt" "log" @@ -17,6 +18,10 @@ const ( // RepoBucket is the bucket in which repos will be cached in the bolt DB RepoBucket string = "repos" + + // HighScoreBucket is the bucket containing the names of the projects with the + // top 100 high scores + HighScoreBucket string = "high_scores" ) // CheckHandler handles the request for checking a repo @@ -63,7 +68,81 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { if b == nil { return fmt.Errorf("Repo bucket not found") } - return b.Put([]byte(repo), respBytes) + + // is this a new repo? if so, increase the count in the high scores bucket later + isNewRepo := b.Get([]byte(repo)) == nil + + err := b.Put([]byte(repo), respBytes) + if err != nil { + return err + } + + // check if we might need to update the high score list + if resp.Files < 100 { + // only repos with >= 100 files are considered for the high score list + return nil + } + + hsb := tx.Bucket([]byte(HighScoreBucket)) + if hsb == nil { + return fmt.Errorf("High score bucket not found") + } + // update total repos count + if isNewRepo { + totalInt := 0 + total := hsb.Get([]byte("total_repos")) + if total != nil { + err = json.Unmarshal(total, totalInt) + if err != nil { + return fmt.Errorf("Could not unmarshal total repos count: %v", err) + } + } + total, err = json.Marshal(totalInt + 1) + if err != nil { + return fmt.Errorf("Could not marshal total repos count: %v", err) + } + hsb.Put([]byte("total_repos"), total) + } + + scoreBytes := hsb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + scores := &scoreHeap{} + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 { + // lowest score on list is higher than this repo's score, so no need to add + return nil + } + // if this repo is already in the list, remove the original entry: + for i := range *scores { + if strings.Compare((*scores)[i].Repo, repo) == 0 { + heap.Remove(scores, i) + break + } + } + // now we can safely push it onto the heap + heap.Push(scores, scoreItem{ + Repo: repo, + Score: resp.Average * 100.0, + Files: resp.Files, + }) + if len(*scores) > 100 { + // trim heap if it's grown to over 100 + *scores = (*scores)[:100] + } + scoreBytes, err = json.Marshal(&scores) + if err != nil { + return err + } + err = hsb.Put([]byte("scores"), scoreBytes) + if err != nil { + return err + } + + return nil }) if err != nil { diff --git a/handlers/checks.go b/handlers/checks.go index f457e01..0bef255 100644 --- a/handlers/checks.go +++ b/handlers/checks.go @@ -1,7 +1,6 @@ package handlers import ( - "encoding/binary" "encoding/json" "errors" "fmt" @@ -49,13 +48,6 @@ func getFromCache(repo string) (checksResp, error) { return resp, nil } -// itob returns an 8-byte big endian representation of v. -func itob(v int) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(v)) - return b -} - type score struct { Name string `json:"name"` Description string `json:"description"` diff --git a/handlers/high_scores.go b/handlers/high_scores.go index 517bc35..94cd9dc 100644 --- a/handlers/high_scores.go +++ b/handlers/high_scores.go @@ -1,8 +1,16 @@ package handlers import ( + "container/heap" + "encoding/json" "fmt" + "log" "net/http" + "text/template" + "time" + + "github.com/boltdb/bolt" + "github.com/dustin/go-humanize" ) func add(x, y int) int { @@ -10,41 +18,55 @@ func add(x, y int) int { } func formatScore(x float64) string { - return fmt.Sprintf("%.2f", x*100) + return fmt.Sprintf("%.2f", x) } // HighScoresHandler handles the stats page func HighScoresHandler(w http.ResponseWriter, r *http.Request) { - // session, err := mgo.Dial(mongoURL) - // if err != nil { - // log.Println("ERROR: could not get collection:", err) - // http.Error(w, err.Error(), 500) - // return - // } - // defer session.Close() - // coll := session.DB(mongoDatabase).C(mongoCollection) - // - // var highScores []struct { - // Repo string - // Files int - // Average float64 - // } - // err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores) - // if err != nil { - // log.Println("ERROR: could not get high scores: ", err) - // http.Error(w, err.Error(), 500) - // return - // } - // - // count, err := coll.Count() - // if err != nil { - // log.Println("ERROR: could not get count of high scores: ", err) - // http.Error(w, err.Error(), 500) - // return - // } - // - // funcs := template.FuncMap{"add": add, "formatScore": formatScore} - // t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) - // - // t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))}) + // write to boltdb + db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + log.Println("Failed to open bolt database: ", err) + return + } + defer db.Close() + + count, scores := 0, &scoreHeap{} + err = db.View(func(tx *bolt.Tx) error { + hsb := tx.Bucket([]byte(HighScoreBucket)) + if hsb == nil { + return fmt.Errorf("High score bucket not found") + } + scoreBytes := hsb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + + total := hsb.Get([]byte("total_repos")) + if total == nil { + count = 0 + } else { + json.Unmarshal(total, &count) + } + return nil + }) + + if err != nil { + log.Println("ERROR: Failed to load high scores from bolt database: ", err) + http.Error(w, err.Error(), 500) + return + } + + funcs := template.FuncMap{"add": add, "formatScore": formatScore} + t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) + + sortedScores := make([]scoreItem, len(*scores)) + for i := range sortedScores { + sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem) + } + + t.Execute(w, map[string]interface{}{"HighScores": sortedScores, "Count": humanize.Comma(int64(count))}) } diff --git a/handlers/score_heap.go b/handlers/score_heap.go new file mode 100644 index 0000000..873aa57 --- /dev/null +++ b/handlers/score_heap.go @@ -0,0 +1,28 @@ +package handlers + +type scoreItem struct { + Repo string `json:"repo"` + Score float64 `json:"score"` + Files int `json:"files"` +} + +// An scoreHeap is a min-heap of ints. +type scoreHeap []scoreItem + +func (h scoreHeap) Len() int { return len(h) } +func (h scoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score } +func (h scoreHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *scoreHeap) Push(x interface{}) { + // Push and Pop use pointer receivers because they modify the slice's length, + // not just its contents. + *h = append(*h, x.(scoreItem)) +} + +func (h *scoreHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} diff --git a/main.go b/main.go index b3e9478..0b0ff04 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,10 @@ func initDB() error { err = db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(handlers.RepoBucket)) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists([]byte(handlers.HighScoreBucket)) return err }) return err diff --git a/templates/high_scores.html b/templates/high_scores.html index 2bb23c1..2f3ea5c 100644 --- a/templates/high_scores.html +++ b/templates/high_scores.html @@ -12,10 +12,10 @@ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - + ga('create', 'UA-58936835-1', 'auto'); ga('send', 'pageview'); - +