add ledis and redis support

This commit is contained in:
Herman Schaaf
2019-07-28 21:00:47 +01:00
parent 05556daf26
commit 5752260ea9
10 changed files with 195 additions and 302 deletions

2
.gitignore vendored
View File

@@ -28,5 +28,5 @@ _testmain.go
_repos
goreportcard.db
goreportcard.DB
var

View File

@@ -1,14 +1,22 @@
package database
import (
"math"
"time"
"github.com/go-redis/redis"
"github.com/siddontang/ledisdb/config"
"github.com/siddontang/ledisdb/ledis"
)
type Database interface {
Get(string) (string, error)
Set(string, string) error
GetRepo(string) (string, error)
SetRepo(repo string, value string) error
SetScore(repo string, score int) error
GetHighScores(n int) ([]string, error)
SetRecentlyViewed(repo string) error
GetMostRecentlyViewed(n int) ([]string, error)
GetRepoCount() (int, error)
Close() error
}
@@ -25,15 +33,57 @@ type ledisDatabase struct {
connection *ledis.DB
}
func (l *ledisDatabase) Get(k string) (string, error) {
func (l *ledisDatabase) GetRepo(k string) (string, error) {
b, _ := l.connection.Get([]byte(k))
return string(b), nil
}
func (l *ledisDatabase) Set(k, v string) error {
func (l *ledisDatabase) SetRepo(k, v string) error {
return l.connection.Set([]byte(k), []byte(v))
}
func (l *ledisDatabase) SetScore(repo string, score int) error {
pair := ledis.ScorePair{Score: int64(score), Member: []byte(repo)}
_, err := l.connection.ZAdd([]byte("scores"), pair)
return err
}
func (l *ledisDatabase) GetHighScores(n int) ([]string, error) {
pairs, err := l.connection.ZRevRange([]byte("scores"), 0, n)
if err != nil {
return nil, err
}
s := make([]string, len(pairs))
for i := range pairs {
s[i] = string(pairs[i].Member)
}
return s, nil
}
func (l *ledisDatabase) SetRecentlyViewed(repo string) error {
pair := ledis.ScorePair{Score: time.Now().UnixNano(), Member: []byte(repo)}
_, err := l.connection.ZAdd([]byte("last_viewed"), pair)
return err
}
func (l *ledisDatabase) GetMostRecentlyViewed(n int) ([]string, error) {
pairs, err := l.connection.ZRevRange([]byte("last_viewed"), 0, n-1)
if err != nil {
return nil, err
}
s := make([]string, len(pairs))
for i := range pairs {
s[i] = string(pairs[i].Member)
}
return s, nil
}
func (l *ledisDatabase) GetRepoCount() (int, error) {
n, err := l.connection.ZCount([]byte("last_viewed"), math.MinInt64, math.MaxInt64)
return int(n), err
}
func (l *ledisDatabase) Close() error {
return nil
}
@@ -69,14 +119,34 @@ func newRedisDatabase(redisHost string) (*redisDatabase, error) {
return &redisDatabase{connection: db}, err
}
func (r *redisDatabase) Get(k string) (string, error) {
func (r *redisDatabase) GetRepo(k string) (string, error) {
return r.connection.Get(k).Result()
}
func (r *redisDatabase) Set(k, v string) error {
func (r *redisDatabase) SetRepo(k, v string) error {
return r.connection.Set(k, v, 0).Err()
}
func (r *redisDatabase) SetScore(repo string, score int) error {
return r.connection.ZAdd("scores", &redis.Z{Member: repo, Score: float64(score)}).Err()
}
func (r *redisDatabase) GetHighScores(n int) ([]string, error) {
return r.connection.ZRevRange("scores", 0, int64(n)).Result()
}
func (r *redisDatabase) SetRecentlyViewed(repo string) error {
return r.connection.ZAdd("last_viewed", &redis.Z{Member: repo, Score: float64(time.Now().UnixNano())}).Err()
}
func (r *redisDatabase) GetMostRecentlyViewed(n int) ([]string, error) {
return r.connection.ZRevRange("last_viewed", 0, int64(n)).Result()
}
func (r *redisDatabase) GetRepoCount() (int, error) {
n, err := r.connection.ZCount("last_viewed", "-inf", "inf").Result()
return int(n), err
}
func (r *redisDatabase) Close() error {
return r.connection.Close()
}

View File

@@ -7,15 +7,20 @@ import (
"strings"
"github.com/gojp/goreportcard/check"
"github.com/gojp/goreportcard/database"
)
type BadgeHandler struct {
DB database.Database
}
func badgePath(grade check.Grade, style string) string {
return fmt.Sprintf("assets/badges/%s_%s.svg", strings.ToLower(string(grade)), strings.ToLower(style))
}
// BadgeHandler handles fetching the badge images
func BadgeHandler(w http.ResponseWriter, r *http.Request, repo string) {
resp, err := newChecksResp(repo, false)
// Handle handles fetching the badge images
func (b *BadgeHandler) Handle(w http.ResponseWriter, r *http.Request, repo string) {
resp, err := newChecksResp(b.DB, repo, false)
// See: http://shields.io/#styles
style := r.URL.Query().Get("style")

View File

@@ -1,31 +1,23 @@
package handlers
import (
"container/heap"
"encoding/json"
"fmt"
"log"
"net/http"
"strings"
"github.com/gojp/goreportcard/database"
"github.com/boltdb/bolt"
"github.com/gojp/goreportcard/download"
)
const (
// DBPath is the relative (or absolute) path to the bolt database file
DBPath string = "goreportcard.db"
type CheckHandler struct {
DB database.Database
}
// RepoBucket is the bucket in which repos will be cached in the bolt DB
RepoBucket string = "repos"
// MetaBucket is the bucket containing the names of the projects with the
// top 100 high scores, and other meta information
MetaBucket string = "meta"
)
// CheckHandler handles the request for checking a repo
func CheckHandler(w http.ResponseWriter, r *http.Request) {
// Handle handles the request for checking a repo
func (c *CheckHandler) Handle(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
repo, err := download.Clean(r.FormValue("repo"))
@@ -38,7 +30,7 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) {
log.Printf("Checking repo %q...", repo)
forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first
_, err = newChecksResp(repo, forceRefresh)
_, err = newChecksResp(c.DB, repo, forceRefresh)
if err != nil {
log.Println("ERROR: from newChecksResp:", err)
http.Error(w, "Could not analyze the repository: "+err.Error(), http.StatusBadRequest)
@@ -53,49 +45,13 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) {
w.Write(b)
}
func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
func updateHighScores(db database.Database, resp checksResp, repo string) error {
// check if we need to update the high score list
if resp.Files < 100 {
// only repos with >= 100 files are considered for the high score list
return nil
}
// start updating high score list
scoreBytes := mb.Get([]byte("scores"))
if scoreBytes == nil {
scoreBytes, _ = json.Marshal([]ScoreHeap{})
}
scores := &ScoreHeap{}
json.Unmarshal(scoreBytes, scores)
heap.Init(scores)
if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 50 {
// lowest score on list is higher than this repo's score, so no need to add, unless
// we do not have 50 high scores yet
return nil
}
// if this repo is already in the list, remove the original entry:
for i := range *scores {
if strings.ToLower((*scores)[i].Repo) == strings.ToLower(repo) {
heap.Remove(scores, i)
break
}
}
// now we can safely push it onto the heap
heap.Push(scores, scoreItem{
Repo: repo,
Score: resp.Average * 100.0,
Files: resp.Files,
})
if len(*scores) > 50 {
// trim heap if it's grown to over 50
*scores = (*scores)[1:51]
}
scoreBytes, err := json.Marshal(&scores)
if err != nil {
return err
}
return mb.Put([]byte("scores"), scoreBytes)
return db.SetScore(repo, int(resp.Average*100))
}
func updateReposCount(mb *bolt.Bucket, repo string) (err error) {
@@ -122,50 +78,10 @@ type recentItem struct {
Repo string
}
func updateRecentlyViewed(mb *bolt.Bucket, repo string) error {
if mb == nil {
return fmt.Errorf("meta bucket not found")
}
b := mb.Get([]byte("recent"))
if b == nil {
b, _ = json.Marshal([]recentItem{})
}
recent := []recentItem{}
json.Unmarshal(b, &recent)
// add it to the slice, if it is not in there already
for i := range recent {
if recent[i].Repo == repo {
return nil
}
}
recent = append(recent, recentItem{Repo: repo})
if len(recent) > 5 {
// trim recent if it's grown to over 5
recent = (recent)[1:6]
}
b, err := json.Marshal(&recent)
if err != nil {
return err
}
return mb.Put([]byte("recent"), b)
func updateRecentlyViewed(db database.Database, repo string) error {
return db.SetRecentlyViewed(repo)
}
//func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool, oldScore *float64) error {
func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool) error {
// fetch meta-bucket
mb := tx.Bucket([]byte(MetaBucket))
if mb == nil {
return fmt.Errorf("high score bucket not found")
}
// update total repos count
if isNewRepo {
err := updateReposCount(mb, repo)
if err != nil {
return err
}
}
return updateHighScores(mb, resp, repo)
func updateMetadata(db database.Database, resp checksResp, repo string) error {
return updateHighScores(db, resp, repo)
}

View File

@@ -7,7 +7,8 @@ import (
"log"
"time"
"github.com/boltdb/bolt"
"github.com/gojp/goreportcard/database"
"github.com/dustin/go-humanize"
"github.com/gojp/goreportcard/check"
"github.com/gojp/goreportcard/download"
@@ -25,35 +26,20 @@ func dirName(repo string) string {
return fmt.Sprintf("_repos/src/%s", repo)
}
func getFromCache(repo string) (checksResp, error) {
// try and fetch from boltdb
db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 3 * time.Second})
if err != nil {
return checksResp{}, fmt.Errorf("failed to open bolt database during GET: %v", err)
}
defer db.Close()
func getFromCache(db database.Database, repo string) (checksResp, error) {
resp := checksResp{}
err = db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return errors.New("No repo bucket")
}
cached := b.Get([]byte(repo))
if cached == nil {
return notFoundError{repo}
}
err = json.Unmarshal(cached, &resp)
if err != nil {
return fmt.Errorf("failed to parse JSON for %q in cache", repo)
}
return nil
})
v, err := db.GetRepo(repo)
if err != nil {
return resp, err
}
if v == "" {
return resp, errors.New("repo not found in cache")
}
err = json.Unmarshal([]byte(v), &resp)
if err != nil {
return resp, fmt.Errorf("failed to parse JSON for %q in cache", repo)
}
resp.LastRefresh = resp.LastRefresh.UTC()
resp.LastRefreshFormatted = resp.LastRefresh.Format(time.UnixDate)
@@ -75,9 +61,9 @@ type checksResp struct {
LastRefreshHumanized string `json:"humanized_last_refresh"`
}
func newChecksResp(repo string, forceRefresh bool) (checksResp, error) {
func newChecksResp(db database.Database, repo string, forceRefresh bool) (checksResp, error) {
if !forceRefresh {
resp, err := getFromCache(repo)
resp, err := getFromCache(db, repo)
if err != nil {
// just log the error and continue
log.Println(err)
@@ -118,60 +104,28 @@ func newChecksResp(repo string, forceRefresh bool) (checksResp, error) {
return checksResp{}, fmt.Errorf("could not marshal json: %v", err)
}
// write to boltdb
db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
cached, err := db.GetRepo(repo)
if err != nil {
return checksResp{}, fmt.Errorf("could not open bolt db: %v", err)
return checksResp{}, fmt.Errorf("could not load from database: %v", err)
}
defer db.Close()
// is this a new repo? if so, increase the count in the high scores bucket later
isNewRepo := false
var oldRepoBytes []byte
err = db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return fmt.Errorf("repo bucket not found")
}
oldRepoBytes = b.Get([]byte(repo))
return nil
})
if err != nil {
log.Println("ERROR getting repo from repo bucket:", err)
}
isNewRepo = oldRepoBytes == nil
isNewRepo := cached == ""
// if this is a new repo, or the user force-refreshed, update the cache
if isNewRepo || forceRefresh {
err = db.Update(func(tx *bolt.Tx) error {
log.Printf("Saving repo %q to cache...", repo)
b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return fmt.Errorf("repo bucket not found")
}
// save repo to cache
err = b.Put([]byte(repo), respBytes)
if err != nil {
return err
}
return updateMetadata(tx, resp, repo, isNewRepo)
})
db.SetRepo(repo, string(respBytes))
if err != nil {
log.Println("Bolt writing error:", err)
log.Println("db writing error when calling SetRepo:", err)
}
err = updateMetadata(db, resp, repo)
if err != nil {
log.Println("db writing error when calling updateMetadata:", err)
}
}
db.Update(func(tx *bolt.Tx) error {
// fetch meta-bucket
mb := tx.Bucket([]byte(MetaBucket))
return updateRecentlyViewed(mb, repo)
})
err = updateRecentlyViewed(db, repo)
if err != nil {
log.Println("db writing error when calling updateRecentlyViewed:", err)
}
return resp, nil
}

View File

@@ -1,18 +1,22 @@
package handlers
import (
"container/heap"
"encoding/json"
"fmt"
"html/template"
"log"
"net/http"
"time"
"github.com/boltdb/bolt"
"github.com/gojp/goreportcard/database"
"github.com/dustin/go-humanize"
)
type scoreItem struct {
Repo string `json:"repo"`
Score float64 `json:"score"`
Files int `json:"files"`
}
func add(x, y int) int {
return x + y
}
@@ -21,57 +25,44 @@ func formatScore(x float64) string {
return fmt.Sprintf("%.2f", x)
}
// HighScoresHandler handles the stats page
func HighScoresHandler(w http.ResponseWriter, r *http.Request) {
// write to boltdb
db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
log.Println("Failed to open bolt database: ", err)
return
}
defer db.Close()
type HighScoresHandler struct {
DB database.Database
}
count, scores := 0, &ScoreHeap{}
err = db.View(func(tx *bolt.Tx) error {
hsb := tx.Bucket([]byte(MetaBucket))
if hsb == nil {
return fmt.Errorf("high score bucket not found")
}
scoreBytes := hsb.Get([]byte("scores"))
if scoreBytes == nil {
scoreBytes, err = json.Marshal([]ScoreHeap{})
if err != nil {
return err
}
}
json.Unmarshal(scoreBytes, scores)
heap.Init(scores)
total := hsb.Get([]byte("total_repos"))
if total == nil {
count = 0
return nil
}
return json.Unmarshal(total, &count)
})
if err != nil {
log.Println("ERROR: Failed to load high scores from bolt database: ", err)
http.Error(w, err.Error(), 500)
return
}
// Handle handles the stats page
func (h *HighScoresHandler) Handle(w http.ResponseWriter, r *http.Request) {
funcs := template.FuncMap{"add": add, "formatScore": formatScore}
t := template.Must(template.New("high_scores.html").Delims("[[", "]]").Funcs(funcs).ParseFiles("templates/high_scores.html", "templates/footer.html"))
sortedScores := make([]scoreItem, len(*scores))
for i := range sortedScores {
sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem)
repos, err := h.DB.GetHighScores(100)
if err != nil {
log.Print("error loading high scores:", err)
return
}
scores := make([]scoreItem, len(repos))
for i, repo := range repos {
cached, err := getFromCache(h.DB, repo)
if err != nil {
log.Print("error loading cached repo:", err)
return
}
scores[i] = scoreItem{
Repo: repo,
Score: cached.Average * 100,
Files: cached.Files,
}
}
count, err := h.DB.GetRepoCount()
if err != nil {
log.Print("error loading repo count:", err)
return
}
t.Execute(w, map[string]interface{}{
"HighScores": sortedScores,
"HighScores": scores,
"Count": humanize.Comma(int64(count)),
"google_analytics_key": googleAnalyticsKey,
})

View File

@@ -1,51 +1,26 @@
package handlers
import (
"encoding/json"
"fmt"
"html/template"
"log"
"net/http"
"time"
"github.com/boltdb/bolt"
"github.com/gojp/goreportcard/database"
)
type HomeHandler struct {
DB database.Database
}
// HomeHandler handles the homepage
func HomeHandler(w http.ResponseWriter, r *http.Request) {
func (h *HomeHandler) Handle(w http.ResponseWriter, r *http.Request) {
if r.URL.Path[1:] == "" {
db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
recentRepos, err := h.DB.GetMostRecentlyViewed(5)
if err != nil {
log.Println("Failed to open bolt database: ", err)
return
log.Println("ERROR: while calling GetMostRecentlyViewed:", err)
recentRepos = []string{}
}
defer db.Close()
recent := &[]recentItem{}
err = db.View(func(tx *bolt.Tx) error {
rb := tx.Bucket([]byte(MetaBucket))
if rb == nil {
return fmt.Errorf("meta bucket not found")
}
b := rb.Get([]byte("recent"))
if b == nil {
b, err = json.Marshal([]recentItem{})
if err != nil {
return err
}
}
json.Unmarshal(b, recent)
return nil
})
var recentRepos = make([]string, len(*recent))
var j = len(*recent) - 1
for _, r := range *recent {
recentRepos[j] = r.Repo
j--
}
t := template.Must(template.New("home.html").Delims("[[", "]]").ParseFiles("templates/home.html", "templates/footer.html"))
t.Execute(w, map[string]interface{}{
"Recent": recentRepos,

View File

@@ -5,6 +5,8 @@ import (
"log"
"net/http"
"github.com/gojp/goreportcard/database"
"flag"
"html/template"
)
@@ -12,11 +14,15 @@ import (
var domain = flag.String("domain", "goreportcard.com", "Domain used for your goreportcard installation")
var googleAnalyticsKey = flag.String("google_analytics_key", "UA-58936835-1", "Google Analytics Account Id")
// ReportHandler handles the report page
func ReportHandler(w http.ResponseWriter, r *http.Request, repo string) {
type ReportHandler struct {
DB database.Database
}
// Handle handles the report page
func (h *ReportHandler) Handle(w http.ResponseWriter, r *http.Request, repo string) {
log.Printf("Displaying report: %q", repo)
t := template.Must(template.New("report.html").Delims("[[", "]]").ParseFiles("templates/report.html", "templates/footer.html"))
resp, err := getFromCache(repo)
resp, err := getFromCache(h.DB, repo)
needToLoad := false
if err != nil {
switch err.(type) {

View File

@@ -1,30 +0,0 @@
package handlers
type scoreItem struct {
Repo string `json:"repo"`
Score float64 `json:"score"`
Files int `json:"files"`
}
// An ScoreHeap is a min-heap of ints.
type ScoreHeap []scoreItem
func (h ScoreHeap) Len() int { return len(h) }
func (h ScoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score }
func (h ScoreHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
// Push onto the heap
func (h *ScoreHeap) Push(x interface{}) {
// Push and Pop use pointer receivers because they modify the slice's length,
// not just its contents.
*h = append(*h, x.(scoreItem))
}
// Pop item off of the heap
func (h *ScoreHeap) Pop() interface{} {
old := *h
n := len(old)
x := old[n-1]
*h = old[0 : n-1]
return x
}

18
main.go
View File

@@ -104,21 +104,27 @@ func main() {
}
// initialize database
_, err := database.GetConnection(redisHost)
db, err := database.GetConnection(redisHost)
if err != nil {
log.Fatal("ERROR: could not connect to db: ", err)
}
m := setupMetrics()
homeHandler := handlers.HomeHandler{DB: db}
checkHandler := handlers.CheckHandler{DB: db}
reportHandler := handlers.ReportHandler{DB: db}
badgeHandler := handlers.BadgeHandler{DB: db}
highScoresHandler := handlers.HighScoresHandler{DB: db}
http.HandleFunc(m.instrument("/assets/", handlers.AssetsHandler))
http.HandleFunc(m.instrument("/favicon.ico", handlers.FaviconHandler))
http.HandleFunc(m.instrument("/checks", handlers.CheckHandler))
http.HandleFunc(m.instrument("/report/", makeHandler("report", handlers.ReportHandler)))
http.HandleFunc(m.instrument("/badge/", makeHandler("badge", handlers.BadgeHandler)))
http.HandleFunc(m.instrument("/high_scores/", handlers.HighScoresHandler))
http.HandleFunc(m.instrument("/checks", checkHandler.Handle))
http.HandleFunc(m.instrument("/report/", makeHandler("report", reportHandler.Handle)))
http.HandleFunc(m.instrument("/badge/", makeHandler("badge", badgeHandler.Handle)))
http.HandleFunc(m.instrument("/high_scores/", highScoresHandler.Handle))
http.HandleFunc(m.instrument("/about/", handlers.AboutHandler))
http.HandleFunc(m.instrument("/", handlers.HomeHandler))
http.HandleFunc(m.instrument("/", homeHandler.Handle))
http.Handle("/metrics", promhttp.Handler())