diff --git a/.gitignore b/.gitignore index 434864e..6fbae7e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,5 @@ _testmain.go _repos goreportcard.db - +goreportcard.DB var \ No newline at end of file diff --git a/database/db.go b/database/db.go index 9a7c493..88fc5f5 100644 --- a/database/db.go +++ b/database/db.go @@ -1,14 +1,22 @@ package database import ( + "math" + "time" + "github.com/go-redis/redis" "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/ledis" ) type Database interface { - Get(string) (string, error) - Set(string, string) error + GetRepo(string) (string, error) + SetRepo(repo string, value string) error + SetScore(repo string, score int) error + GetHighScores(n int) ([]string, error) + SetRecentlyViewed(repo string) error + GetMostRecentlyViewed(n int) ([]string, error) + GetRepoCount() (int, error) Close() error } @@ -25,15 +33,57 @@ type ledisDatabase struct { connection *ledis.DB } -func (l *ledisDatabase) Get(k string) (string, error) { +func (l *ledisDatabase) GetRepo(k string) (string, error) { b, _ := l.connection.Get([]byte(k)) return string(b), nil } -func (l *ledisDatabase) Set(k, v string) error { +func (l *ledisDatabase) SetRepo(k, v string) error { return l.connection.Set([]byte(k), []byte(v)) } +func (l *ledisDatabase) SetScore(repo string, score int) error { + pair := ledis.ScorePair{Score: int64(score), Member: []byte(repo)} + _, err := l.connection.ZAdd([]byte("scores"), pair) + return err +} +func (l *ledisDatabase) GetHighScores(n int) ([]string, error) { + pairs, err := l.connection.ZRevRange([]byte("scores"), 0, n) + if err != nil { + return nil, err + } + + s := make([]string, len(pairs)) + for i := range pairs { + s[i] = string(pairs[i].Member) + } + return s, nil +} + +func (l *ledisDatabase) SetRecentlyViewed(repo string) error { + pair := ledis.ScorePair{Score: time.Now().UnixNano(), Member: []byte(repo)} + _, err := l.connection.ZAdd([]byte("last_viewed"), pair) + return err +} + +func (l *ledisDatabase) GetMostRecentlyViewed(n int) ([]string, error) { + pairs, err := l.connection.ZRevRange([]byte("last_viewed"), 0, n-1) + if err != nil { + return nil, err + } + + s := make([]string, len(pairs)) + for i := range pairs { + s[i] = string(pairs[i].Member) + } + return s, nil +} + +func (l *ledisDatabase) GetRepoCount() (int, error) { + n, err := l.connection.ZCount([]byte("last_viewed"), math.MinInt64, math.MaxInt64) + return int(n), err +} + func (l *ledisDatabase) Close() error { return nil } @@ -69,14 +119,34 @@ func newRedisDatabase(redisHost string) (*redisDatabase, error) { return &redisDatabase{connection: db}, err } -func (r *redisDatabase) Get(k string) (string, error) { +func (r *redisDatabase) GetRepo(k string) (string, error) { return r.connection.Get(k).Result() } -func (r *redisDatabase) Set(k, v string) error { +func (r *redisDatabase) SetRepo(k, v string) error { return r.connection.Set(k, v, 0).Err() } +func (r *redisDatabase) SetScore(repo string, score int) error { + return r.connection.ZAdd("scores", &redis.Z{Member: repo, Score: float64(score)}).Err() +} + +func (r *redisDatabase) GetHighScores(n int) ([]string, error) { + return r.connection.ZRevRange("scores", 0, int64(n)).Result() +} + +func (r *redisDatabase) SetRecentlyViewed(repo string) error { + return r.connection.ZAdd("last_viewed", &redis.Z{Member: repo, Score: float64(time.Now().UnixNano())}).Err() +} + +func (r *redisDatabase) GetMostRecentlyViewed(n int) ([]string, error) { + return r.connection.ZRevRange("last_viewed", 0, int64(n)).Result() +} + +func (r *redisDatabase) GetRepoCount() (int, error) { + n, err := r.connection.ZCount("last_viewed", "-inf", "inf").Result() + return int(n), err +} func (r *redisDatabase) Close() error { return r.connection.Close() } diff --git a/handlers/badge.go b/handlers/badge.go index 861a204..e690fd6 100644 --- a/handlers/badge.go +++ b/handlers/badge.go @@ -7,15 +7,20 @@ import ( "strings" "github.com/gojp/goreportcard/check" + "github.com/gojp/goreportcard/database" ) +type BadgeHandler struct { + DB database.Database +} + func badgePath(grade check.Grade, style string) string { return fmt.Sprintf("assets/badges/%s_%s.svg", strings.ToLower(string(grade)), strings.ToLower(style)) } -// BadgeHandler handles fetching the badge images -func BadgeHandler(w http.ResponseWriter, r *http.Request, repo string) { - resp, err := newChecksResp(repo, false) +// Handle handles fetching the badge images +func (b *BadgeHandler) Handle(w http.ResponseWriter, r *http.Request, repo string) { + resp, err := newChecksResp(b.DB, repo, false) // See: http://shields.io/#styles style := r.URL.Query().Get("style") diff --git a/handlers/check.go b/handlers/check.go index 0563a16..0580a1b 100644 --- a/handlers/check.go +++ b/handlers/check.go @@ -1,31 +1,23 @@ package handlers import ( - "container/heap" "encoding/json" "fmt" "log" "net/http" - "strings" + + "github.com/gojp/goreportcard/database" "github.com/boltdb/bolt" "github.com/gojp/goreportcard/download" ) -const ( - // DBPath is the relative (or absolute) path to the bolt database file - DBPath string = "goreportcard.db" +type CheckHandler struct { + DB database.Database +} - // RepoBucket is the bucket in which repos will be cached in the bolt DB - RepoBucket string = "repos" - - // MetaBucket is the bucket containing the names of the projects with the - // top 100 high scores, and other meta information - MetaBucket string = "meta" -) - -// CheckHandler handles the request for checking a repo -func CheckHandler(w http.ResponseWriter, r *http.Request) { +// Handle handles the request for checking a repo +func (c *CheckHandler) Handle(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") repo, err := download.Clean(r.FormValue("repo")) @@ -38,7 +30,7 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Checking repo %q...", repo) forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first - _, err = newChecksResp(repo, forceRefresh) + _, err = newChecksResp(c.DB, repo, forceRefresh) if err != nil { log.Println("ERROR: from newChecksResp:", err) http.Error(w, "Could not analyze the repository: "+err.Error(), http.StatusBadRequest) @@ -53,49 +45,13 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) { w.Write(b) } -func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error { +func updateHighScores(db database.Database, resp checksResp, repo string) error { // check if we need to update the high score list if resp.Files < 100 { // only repos with >= 100 files are considered for the high score list return nil } - - // start updating high score list - scoreBytes := mb.Get([]byte("scores")) - if scoreBytes == nil { - scoreBytes, _ = json.Marshal([]ScoreHeap{}) - } - scores := &ScoreHeap{} - json.Unmarshal(scoreBytes, scores) - - heap.Init(scores) - if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 50 { - // lowest score on list is higher than this repo's score, so no need to add, unless - // we do not have 50 high scores yet - return nil - } - // if this repo is already in the list, remove the original entry: - for i := range *scores { - if strings.ToLower((*scores)[i].Repo) == strings.ToLower(repo) { - heap.Remove(scores, i) - break - } - } - // now we can safely push it onto the heap - heap.Push(scores, scoreItem{ - Repo: repo, - Score: resp.Average * 100.0, - Files: resp.Files, - }) - if len(*scores) > 50 { - // trim heap if it's grown to over 50 - *scores = (*scores)[1:51] - } - scoreBytes, err := json.Marshal(&scores) - if err != nil { - return err - } - return mb.Put([]byte("scores"), scoreBytes) + return db.SetScore(repo, int(resp.Average*100)) } func updateReposCount(mb *bolt.Bucket, repo string) (err error) { @@ -122,50 +78,10 @@ type recentItem struct { Repo string } -func updateRecentlyViewed(mb *bolt.Bucket, repo string) error { - if mb == nil { - return fmt.Errorf("meta bucket not found") - } - b := mb.Get([]byte("recent")) - if b == nil { - b, _ = json.Marshal([]recentItem{}) - } - recent := []recentItem{} - json.Unmarshal(b, &recent) - - // add it to the slice, if it is not in there already - for i := range recent { - if recent[i].Repo == repo { - return nil - } - } - - recent = append(recent, recentItem{Repo: repo}) - if len(recent) > 5 { - // trim recent if it's grown to over 5 - recent = (recent)[1:6] - } - b, err := json.Marshal(&recent) - if err != nil { - return err - } - return mb.Put([]byte("recent"), b) +func updateRecentlyViewed(db database.Database, repo string) error { + return db.SetRecentlyViewed(repo) } -//func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool, oldScore *float64) error { -func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool) error { - // fetch meta-bucket - mb := tx.Bucket([]byte(MetaBucket)) - if mb == nil { - return fmt.Errorf("high score bucket not found") - } - // update total repos count - if isNewRepo { - err := updateReposCount(mb, repo) - if err != nil { - return err - } - } - - return updateHighScores(mb, resp, repo) +func updateMetadata(db database.Database, resp checksResp, repo string) error { + return updateHighScores(db, resp, repo) } diff --git a/handlers/checks.go b/handlers/checks.go index 8942df6..75a2011 100644 --- a/handlers/checks.go +++ b/handlers/checks.go @@ -7,7 +7,8 @@ import ( "log" "time" - "github.com/boltdb/bolt" + "github.com/gojp/goreportcard/database" + "github.com/dustin/go-humanize" "github.com/gojp/goreportcard/check" "github.com/gojp/goreportcard/download" @@ -25,35 +26,20 @@ func dirName(repo string) string { return fmt.Sprintf("_repos/src/%s", repo) } -func getFromCache(repo string) (checksResp, error) { - // try and fetch from boltdb - db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 3 * time.Second}) - if err != nil { - return checksResp{}, fmt.Errorf("failed to open bolt database during GET: %v", err) - } - defer db.Close() - +func getFromCache(db database.Database, repo string) (checksResp, error) { resp := checksResp{} - err = db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(RepoBucket)) - if b == nil { - return errors.New("No repo bucket") - } - cached := b.Get([]byte(repo)) - if cached == nil { - return notFoundError{repo} - } - - err = json.Unmarshal(cached, &resp) - if err != nil { - return fmt.Errorf("failed to parse JSON for %q in cache", repo) - } - return nil - }) - + v, err := db.GetRepo(repo) if err != nil { return resp, err } + if v == "" { + return resp, errors.New("repo not found in cache") + } + + err = json.Unmarshal([]byte(v), &resp) + if err != nil { + return resp, fmt.Errorf("failed to parse JSON for %q in cache", repo) + } resp.LastRefresh = resp.LastRefresh.UTC() resp.LastRefreshFormatted = resp.LastRefresh.Format(time.UnixDate) @@ -75,9 +61,9 @@ type checksResp struct { LastRefreshHumanized string `json:"humanized_last_refresh"` } -func newChecksResp(repo string, forceRefresh bool) (checksResp, error) { +func newChecksResp(db database.Database, repo string, forceRefresh bool) (checksResp, error) { if !forceRefresh { - resp, err := getFromCache(repo) + resp, err := getFromCache(db, repo) if err != nil { // just log the error and continue log.Println(err) @@ -118,60 +104,28 @@ func newChecksResp(repo string, forceRefresh bool) (checksResp, error) { return checksResp{}, fmt.Errorf("could not marshal json: %v", err) } - // write to boltdb - db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) + cached, err := db.GetRepo(repo) if err != nil { - return checksResp{}, fmt.Errorf("could not open bolt db: %v", err) + return checksResp{}, fmt.Errorf("could not load from database: %v", err) } - defer db.Close() - - // is this a new repo? if so, increase the count in the high scores bucket later - isNewRepo := false - var oldRepoBytes []byte - err = db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(RepoBucket)) - if b == nil { - return fmt.Errorf("repo bucket not found") - } - oldRepoBytes = b.Get([]byte(repo)) - return nil - }) - if err != nil { - log.Println("ERROR getting repo from repo bucket:", err) - } - - isNewRepo = oldRepoBytes == nil + isNewRepo := cached == "" // if this is a new repo, or the user force-refreshed, update the cache if isNewRepo || forceRefresh { - err = db.Update(func(tx *bolt.Tx) error { - log.Printf("Saving repo %q to cache...", repo) - - b := tx.Bucket([]byte(RepoBucket)) - if b == nil { - return fmt.Errorf("repo bucket not found") - } - - // save repo to cache - err = b.Put([]byte(repo), respBytes) - if err != nil { - return err - } - - return updateMetadata(tx, resp, repo, isNewRepo) - }) - + db.SetRepo(repo, string(respBytes)) if err != nil { - log.Println("Bolt writing error:", err) + log.Println("db writing error when calling SetRepo:", err) + } + err = updateMetadata(db, resp, repo) + if err != nil { + log.Println("db writing error when calling updateMetadata:", err) } - } - db.Update(func(tx *bolt.Tx) error { - // fetch meta-bucket - mb := tx.Bucket([]byte(MetaBucket)) - return updateRecentlyViewed(mb, repo) - }) + err = updateRecentlyViewed(db, repo) + if err != nil { + log.Println("db writing error when calling updateRecentlyViewed:", err) + } return resp, nil } diff --git a/handlers/high_scores.go b/handlers/high_scores.go index b3a03b0..858f952 100644 --- a/handlers/high_scores.go +++ b/handlers/high_scores.go @@ -1,18 +1,22 @@ package handlers import ( - "container/heap" - "encoding/json" "fmt" "html/template" "log" "net/http" - "time" - "github.com/boltdb/bolt" + "github.com/gojp/goreportcard/database" + "github.com/dustin/go-humanize" ) +type scoreItem struct { + Repo string `json:"repo"` + Score float64 `json:"score"` + Files int `json:"files"` +} + func add(x, y int) int { return x + y } @@ -21,57 +25,44 @@ func formatScore(x float64) string { return fmt.Sprintf("%.2f", x) } -// HighScoresHandler handles the stats page -func HighScoresHandler(w http.ResponseWriter, r *http.Request) { - // write to boltdb - db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - log.Println("Failed to open bolt database: ", err) - return - } - defer db.Close() +type HighScoresHandler struct { + DB database.Database +} - count, scores := 0, &ScoreHeap{} - err = db.View(func(tx *bolt.Tx) error { - hsb := tx.Bucket([]byte(MetaBucket)) - if hsb == nil { - return fmt.Errorf("high score bucket not found") - } - scoreBytes := hsb.Get([]byte("scores")) - if scoreBytes == nil { - scoreBytes, err = json.Marshal([]ScoreHeap{}) - if err != nil { - return err - } - } - json.Unmarshal(scoreBytes, scores) - - heap.Init(scores) - - total := hsb.Get([]byte("total_repos")) - if total == nil { - count = 0 - return nil - } - return json.Unmarshal(total, &count) - }) - - if err != nil { - log.Println("ERROR: Failed to load high scores from bolt database: ", err) - http.Error(w, err.Error(), 500) - return - } +// Handle handles the stats page +func (h *HighScoresHandler) Handle(w http.ResponseWriter, r *http.Request) { funcs := template.FuncMap{"add": add, "formatScore": formatScore} t := template.Must(template.New("high_scores.html").Delims("[[", "]]").Funcs(funcs).ParseFiles("templates/high_scores.html", "templates/footer.html")) - sortedScores := make([]scoreItem, len(*scores)) - for i := range sortedScores { - sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem) + repos, err := h.DB.GetHighScores(100) + if err != nil { + log.Print("error loading high scores:", err) + return + } + + scores := make([]scoreItem, len(repos)) + for i, repo := range repos { + cached, err := getFromCache(h.DB, repo) + if err != nil { + log.Print("error loading cached repo:", err) + return + } + scores[i] = scoreItem{ + Repo: repo, + Score: cached.Average * 100, + Files: cached.Files, + } + } + + count, err := h.DB.GetRepoCount() + if err != nil { + log.Print("error loading repo count:", err) + return } t.Execute(w, map[string]interface{}{ - "HighScores": sortedScores, + "HighScores": scores, "Count": humanize.Comma(int64(count)), "google_analytics_key": googleAnalyticsKey, }) diff --git a/handlers/home.go b/handlers/home.go index 5845ea1..bb56e1a 100644 --- a/handlers/home.go +++ b/handlers/home.go @@ -1,51 +1,26 @@ package handlers import ( - "encoding/json" - "fmt" "html/template" "log" "net/http" - "time" - "github.com/boltdb/bolt" + "github.com/gojp/goreportcard/database" ) +type HomeHandler struct { + DB database.Database +} + // HomeHandler handles the homepage -func HomeHandler(w http.ResponseWriter, r *http.Request) { +func (h *HomeHandler) Handle(w http.ResponseWriter, r *http.Request) { if r.URL.Path[1:] == "" { - db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) + + recentRepos, err := h.DB.GetMostRecentlyViewed(5) if err != nil { - log.Println("Failed to open bolt database: ", err) - return + log.Println("ERROR: while calling GetMostRecentlyViewed:", err) + recentRepos = []string{} } - defer db.Close() - - recent := &[]recentItem{} - err = db.View(func(tx *bolt.Tx) error { - rb := tx.Bucket([]byte(MetaBucket)) - if rb == nil { - return fmt.Errorf("meta bucket not found") - } - b := rb.Get([]byte("recent")) - if b == nil { - b, err = json.Marshal([]recentItem{}) - if err != nil { - return err - } - } - json.Unmarshal(b, recent) - - return nil - }) - - var recentRepos = make([]string, len(*recent)) - var j = len(*recent) - 1 - for _, r := range *recent { - recentRepos[j] = r.Repo - j-- - } - t := template.Must(template.New("home.html").Delims("[[", "]]").ParseFiles("templates/home.html", "templates/footer.html")) t.Execute(w, map[string]interface{}{ "Recent": recentRepos, diff --git a/handlers/report.go b/handlers/report.go index bfe478b..6ddbafb 100644 --- a/handlers/report.go +++ b/handlers/report.go @@ -5,6 +5,8 @@ import ( "log" "net/http" + "github.com/gojp/goreportcard/database" + "flag" "html/template" ) @@ -12,11 +14,15 @@ import ( var domain = flag.String("domain", "goreportcard.com", "Domain used for your goreportcard installation") var googleAnalyticsKey = flag.String("google_analytics_key", "UA-58936835-1", "Google Analytics Account Id") -// ReportHandler handles the report page -func ReportHandler(w http.ResponseWriter, r *http.Request, repo string) { +type ReportHandler struct { + DB database.Database +} + +// Handle handles the report page +func (h *ReportHandler) Handle(w http.ResponseWriter, r *http.Request, repo string) { log.Printf("Displaying report: %q", repo) t := template.Must(template.New("report.html").Delims("[[", "]]").ParseFiles("templates/report.html", "templates/footer.html")) - resp, err := getFromCache(repo) + resp, err := getFromCache(h.DB, repo) needToLoad := false if err != nil { switch err.(type) { diff --git a/handlers/score_heap.go b/handlers/score_heap.go deleted file mode 100644 index 509ce92..0000000 --- a/handlers/score_heap.go +++ /dev/null @@ -1,30 +0,0 @@ -package handlers - -type scoreItem struct { - Repo string `json:"repo"` - Score float64 `json:"score"` - Files int `json:"files"` -} - -// An ScoreHeap is a min-heap of ints. -type ScoreHeap []scoreItem - -func (h ScoreHeap) Len() int { return len(h) } -func (h ScoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score } -func (h ScoreHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } - -// Push onto the heap -func (h *ScoreHeap) Push(x interface{}) { - // Push and Pop use pointer receivers because they modify the slice's length, - // not just its contents. - *h = append(*h, x.(scoreItem)) -} - -// Pop item off of the heap -func (h *ScoreHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} diff --git a/main.go b/main.go index 7139f3d..73c1a5f 100644 --- a/main.go +++ b/main.go @@ -104,21 +104,27 @@ func main() { } // initialize database - _, err := database.GetConnection(redisHost) + db, err := database.GetConnection(redisHost) if err != nil { log.Fatal("ERROR: could not connect to db: ", err) } m := setupMetrics() + homeHandler := handlers.HomeHandler{DB: db} + checkHandler := handlers.CheckHandler{DB: db} + reportHandler := handlers.ReportHandler{DB: db} + badgeHandler := handlers.BadgeHandler{DB: db} + highScoresHandler := handlers.HighScoresHandler{DB: db} + http.HandleFunc(m.instrument("/assets/", handlers.AssetsHandler)) http.HandleFunc(m.instrument("/favicon.ico", handlers.FaviconHandler)) - http.HandleFunc(m.instrument("/checks", handlers.CheckHandler)) - http.HandleFunc(m.instrument("/report/", makeHandler("report", handlers.ReportHandler))) - http.HandleFunc(m.instrument("/badge/", makeHandler("badge", handlers.BadgeHandler))) - http.HandleFunc(m.instrument("/high_scores/", handlers.HighScoresHandler)) + http.HandleFunc(m.instrument("/checks", checkHandler.Handle)) + http.HandleFunc(m.instrument("/report/", makeHandler("report", reportHandler.Handle))) + http.HandleFunc(m.instrument("/badge/", makeHandler("badge", badgeHandler.Handle))) + http.HandleFunc(m.instrument("/high_scores/", highScoresHandler.Handle)) http.HandleFunc(m.instrument("/about/", handlers.AboutHandler)) - http.HandleFunc(m.instrument("/", handlers.HomeHandler)) + http.HandleFunc(m.instrument("/", homeHandler.Handle)) http.Handle("/metrics", promhttp.Handler())