diff --git a/.gitignore b/.gitignore index f7ed1d4..448b03d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ _testmain.go *.swp *.DS_Store repos + +goreportcard.db diff --git a/.travis.yml b/.travis.yml index a8e2f9f..4432489 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: go go: 1.5 install: + - go get github.com/boltdb/bolt - go get github.com/fzipp/gocyclo - go get github.com/golang/lint - go get golang.org/x/tools/cmd/vet diff --git a/handlers/check.go b/handlers/check.go index d4409c4..a67879d 100644 --- a/handlers/check.go +++ b/handlers/check.go @@ -1,13 +1,27 @@ package handlers import ( + "container/heap" "encoding/json" + "fmt" "log" "net/http" "strings" + "time" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "github.com/boltdb/bolt" +) + +const ( + // DBPath is the relative (or absolute) path to the bolt database file + DBPath string = "goreportcard.db" + + // RepoBucket is the bucket in which repos will be cached in the bolt DB + RepoBucket string = "repos" + + // MetaBucket is the bucket containing the names of the projects with the + // top 100 high scores, and other meta information + MetaBucket string = "meta" ) // CheckHandler handles the request for checking a repo @@ -21,7 +35,7 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte("We've decided to omit results for the Go repository because it has lots of test files that (purposely) don't pass our checks. Go gets an A+ in our books though!")) return } - forceRefresh := r.Method != "GET" // if this is a GET request, try fetch from cached version in mongo first + forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first resp, err := newChecksResp(repo, forceRefresh) if err != nil { log.Println("ERROR: ", err) @@ -31,26 +45,143 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { return } - b, err := json.Marshal(resp) + respBytes, err := json.Marshal(resp) if err != nil { log.Println("ERROR: could not marshal json:", err) http.Error(w, err.Error(), 500) return } - w.Write(b) + w.Write(respBytes) - // write to mongo - session, err := mgo.Dial(mongoURL) + // write to boltdb + db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { - log.Println("Failed to get mongo collection: ", err) + log.Println("Failed to open bolt database: ", err) return } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) - log.Printf("Upserting repo %s...", repo) - _, err = coll.Upsert(bson.M{"repo": repo}, resp) + defer db.Close() + + log.Printf("Saving repo %q to cache...", repo) + + // is this a new repo? if so, increase the count in the high scores bucket later + isNewRepo := false + err = db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(RepoBucket)) + if b == nil { + return fmt.Errorf("repo bucket not found") + } + isNewRepo = b.Get([]byte(repo)) == nil + return nil + }) if err != nil { - log.Println("Mongo writing error:", err) - return + log.Println(err) } + + // if this is a new repo, or the user force-refreshed, update the cache + if isNewRepo || forceRefresh { + err = db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(RepoBucket)) + if b == nil { + return fmt.Errorf("repo bucket not found") + } + + // save repo to cache + err = b.Put([]byte(repo), respBytes) + if err != nil { + return err + } + + // fetch meta-bucket + mb := tx.Bucket([]byte(MetaBucket)) + if mb == nil { + return fmt.Errorf("high score bucket not found") + } + + // update total repos count + if isNewRepo { + err = updateReposCount(mb, resp, repo) + if err != nil { + return err + } + } + + return updateHighScores(mb, resp, repo) + }) + + if err != nil { + log.Println("Bolt writing error:", err) + } + } + + return +} + +func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error { + // check if we need to update the high score list + if resp.Files < 100 { + // only repos with >= 100 files are considered for the high score list + return nil + } + + // start updating high score list + scoreBytes := mb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + scores := &scoreHeap{} + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 100 { + // lowest score on list is higher than this repo's score, so no need to add, unless + // we do not have 100 high scores yet + return nil + } + // if this repo is already in the list, remove the original entry: + for i := range *scores { + if (*scores)[i].Repo == repo { + heap.Remove(scores, i) + break + } + } + // now we can safely push it onto the heap + heap.Push(scores, scoreItem{ + Repo: repo, + Score: resp.Average * 100.0, + Files: resp.Files, + }) + if len(*scores) > 100 { + // trim heap if it's grown to over 100 + *scores = (*scores)[:100] + } + scoreBytes, err := json.Marshal(&scores) + if err != nil { + return err + } + err = mb.Put([]byte("scores"), scoreBytes) + if err != nil { + return err + } + + return nil +} + +func updateReposCount(mb *bolt.Bucket, resp checksResp, repo string) (err error) { + log.Printf("New repo %q, adding to repo count...", repo) + totalInt := 0 + total := mb.Get([]byte("total_repos")) + if total != nil { + err = json.Unmarshal(total, &totalInt) + if err != nil { + return fmt.Errorf("could not unmarshal total repos count: %v", err) + } + } + totalInt++ // increase repo count + total, err = json.Marshal(totalInt) + if err != nil { + return fmt.Errorf("could not marshal total repos count: %v", err) + } + mb.Put([]byte("total_repos"), total) + log.Println("Repo count is now", totalInt) + return nil } diff --git a/handlers/checks.go b/handlers/checks.go index bf19dc9..0bef255 100644 --- a/handlers/checks.go +++ b/handlers/checks.go @@ -1,6 +1,8 @@ package handlers import ( + "encoding/json" + "errors" "fmt" "log" "os" @@ -8,33 +10,41 @@ import ( "strings" "time" + "github.com/boltdb/bolt" "github.com/gojp/goreportcard/check" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -var ( - mongoURL = "mongodb://127.0.0.1:27017" - mongoDatabase = "goreportcard" - mongoCollection = "reports" ) func getFromCache(repo string) (checksResp, error) { - // try and fetch from mongo - session, err := mgo.Dial(mongoURL) + // try and fetch from boltdb + db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { - return checksResp{}, fmt.Errorf("Failed to get mongo collection during GET: %v", err) + return checksResp{}, fmt.Errorf("Failed to open bolt database during GET: %v", err) } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) + defer db.Close() + resp := checksResp{} - err = coll.Find(bson.M{"repo": repo}).One(&resp) + err = db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(RepoBucket)) + if b == nil { + return errors.New("No repo bucket") + } + cached := b.Get([]byte(repo)) + if cached == nil { + return fmt.Errorf("%q not found in cache", repo) + } + + err = json.Unmarshal(cached, &resp) + if err != nil { + return fmt.Errorf("failed to parse JSON for %q in cache", repo) + } + return nil + }) + if err != nil { - return checksResp{}, fmt.Errorf("Failed to fetch %q from mongo: %v", repo, err) + return resp, err } resp.LastRefresh = resp.LastRefresh.UTC() - return resp, nil } diff --git a/handlers/high_scores.go b/handlers/high_scores.go index 7021188..eec581e 100644 --- a/handlers/high_scores.go +++ b/handlers/high_scores.go @@ -1,14 +1,16 @@ package handlers import ( + "container/heap" + "encoding/json" "fmt" "log" "net/http" "text/template" + "time" + "github.com/boltdb/bolt" "github.com/dustin/go-humanize" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" ) func add(x, y int) int { @@ -16,35 +18,44 @@ func add(x, y int) int { } func formatScore(x float64) string { - return fmt.Sprintf("%.2f", x*100) + return fmt.Sprintf("%.2f", x) } // HighScoresHandler handles the stats page func HighScoresHandler(w http.ResponseWriter, r *http.Request) { - session, err := mgo.Dial(mongoURL) + // write to boltdb + db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { - log.Println("ERROR: could not get collection:", err) - http.Error(w, err.Error(), 500) + log.Println("Failed to open bolt database: ", err) return } - defer session.Close() - coll := session.DB(mongoDatabase).C(mongoCollection) + defer db.Close() - var highScores []struct { - Repo string - Files int - Average float64 - } - err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores) - if err != nil { - log.Println("ERROR: could not get high scores: ", err) - http.Error(w, err.Error(), 500) - return - } + count, scores := 0, &scoreHeap{} + err = db.View(func(tx *bolt.Tx) error { + hsb := tx.Bucket([]byte(MetaBucket)) + if hsb == nil { + return fmt.Errorf("High score bucket not found") + } + scoreBytes := hsb.Get([]byte("scores")) + if scoreBytes == nil { + scoreBytes, _ = json.Marshal([]scoreHeap{}) + } + json.Unmarshal(scoreBytes, scores) + + heap.Init(scores) + + total := hsb.Get([]byte("total_repos")) + if total == nil { + count = 0 + } else { + json.Unmarshal(total, &count) + } + return nil + }) - count, err := coll.Count() if err != nil { - log.Println("ERROR: could not get count of high scores: ", err) + log.Println("ERROR: Failed to load high scores from bolt database: ", err) http.Error(w, err.Error(), 500) return } @@ -52,5 +63,10 @@ func HighScoresHandler(w http.ResponseWriter, r *http.Request) { funcs := template.FuncMap{"add": add, "formatScore": formatScore} t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html")) - t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))}) + sortedScores := make([]scoreItem, len(*scores)) + for i := range sortedScores { + sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem) + } + + t.Execute(w, map[string]interface{}{"HighScores": sortedScores, "Count": humanize.Comma(int64(count))}) } diff --git a/handlers/score_heap.go b/handlers/score_heap.go new file mode 100644 index 0000000..873aa57 --- /dev/null +++ b/handlers/score_heap.go @@ -0,0 +1,28 @@ +package handlers + +type scoreItem struct { + Repo string `json:"repo"` + Score float64 `json:"score"` + Files int `json:"files"` +} + +// An scoreHeap is a min-heap of ints. +type scoreHeap []scoreItem + +func (h scoreHeap) Len() int { return len(h) } +func (h scoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score } +func (h scoreHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *scoreHeap) Push(x interface{}) { + // Push and Pop use pointer receivers because they modify the slice's length, + // not just its contents. + *h = append(*h, x.(scoreItem)) +} + +func (h *scoreHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} diff --git a/main.go b/main.go index 01ea93b..1f5ee30 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,9 @@ import ( "net/http" "os" "regexp" + "time" + "github.com/boltdb/bolt" "github.com/gojp/goreportcard/handlers" ) @@ -31,11 +33,36 @@ func makeHandler(name string, fn func(http.ResponseWriter, *http.Request, string } } +// initDB opens the bolt database file (or creates it if it does not exist), and creates +// a bucket for saving the repos, also only if it does not exist. +func initDB() error { + db, err := bolt.Open(handlers.DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return err + } + defer db.Close() + + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(handlers.RepoBucket)) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists([]byte(handlers.MetaBucket)) + return err + }) + return err +} + func main() { if err := os.MkdirAll("repos/src/github.com", 0755); err != nil && !os.IsExist(err) { log.Fatal("ERROR: could not create repos dir: ", err) } + // initialize database + if err := initDB(); err != nil { + log.Fatal("ERROR: could not open bolt db: ", err) + } + http.HandleFunc("/assets/", handlers.AssetsHandler) http.HandleFunc("/checks", handlers.CheckHandler) http.HandleFunc("/report/", makeHandler("report", handlers.ReportHandler)) diff --git a/templates/high_scores.html b/templates/high_scores.html index 2bb23c1..fcf8530 100644 --- a/templates/high_scores.html +++ b/templates/high_scores.html @@ -12,10 +12,10 @@ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - + ga('create', 'UA-58936835-1', 'auto'); ga('send', 'pageview'); - +