Merge pull request #65 from gojp/bolt

First pass of writing to and loading from bolt

.gitignore

@@ -26,3 +26,5 @@ _testmain.go
 *.swp
 *.DS_Store
 repos
+
+goreportcard.db

.travis.yml

@@ -3,6 +3,7 @@ language: go
 go: 1.5
 
 install:
+  - go get github.com/boltdb/bolt
   - go get github.com/fzipp/gocyclo
   - go get github.com/golang/lint
   - go get golang.org/x/tools/cmd/vet

handlers/check.go

@@ -1,13 +1,27 @@
 package handlers
 
 import (
+	"container/heap"
 	"encoding/json"
+	"fmt"
 	"log"
 	"net/http"
 	"strings"
+	"time"
 
-	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// DBPath is the relative (or absolute) path to the bolt database file
+	DBPath string = "goreportcard.db"
+
+	// RepoBucket is the bucket in which repos will be cached in the bolt DB
+	RepoBucket string = "repos"
+
+	// MetaBucket is the bucket containing the names of the projects with the
+	// top 100 high scores, and other meta information
+	MetaBucket string = "meta"
 )
 
 // CheckHandler handles the request for checking a repo
@@ -21,7 +35,7 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("We've decided to omit results for the Go repository because it has lots of test files that (purposely) don't pass our checks. Go gets an A+ in our books though!"))
 		return
 	}
-	forceRefresh := r.Method != "GET" // if this is a GET request, try fetch from cached version in mongo first
+	forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first
 	resp, err := newChecksResp(repo, forceRefresh)
 	if err != nil {
 		log.Println("ERROR: ", err)
@@ -31,26 +45,143 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	b, err := json.Marshal(resp)
+	respBytes, err := json.Marshal(resp)
 	if err != nil {
 		log.Println("ERROR: could not marshal json:", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
-	w.Write(b)
+	w.Write(respBytes)
 
-	// write to mongo
-	session, err := mgo.Dial(mongoURL)
+	// write to boltdb
+	db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
-		log.Println("Failed to get mongo collection: ", err)
+		log.Println("Failed to open bolt database: ", err)
 		return
 	}
-	defer session.Close()
-	coll := session.DB(mongoDatabase).C(mongoCollection)
-	log.Printf("Upserting repo %s...", repo)
-	_, err = coll.Upsert(bson.M{"repo": repo}, resp)
+	defer db.Close()
+
+	log.Printf("Saving repo %q to cache...", repo)
+
+	// is this a new repo? if so, increase the count in the high scores bucket later
+	isNewRepo := false
+	err = db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(RepoBucket))
+		if b == nil {
+			return fmt.Errorf("repo bucket not found")
+		}
+		isNewRepo = b.Get([]byte(repo)) == nil
+		return nil
+	})
 	if err != nil {
-		log.Println("Mongo writing error:", err)
-		return
+		log.Println(err)
 	}
+
+	// if this is a new repo, or the user force-refreshed, update the cache
+	if isNewRepo || forceRefresh {
+		err = db.Update(func(tx *bolt.Tx) error {
+			b := tx.Bucket([]byte(RepoBucket))
+			if b == nil {
+				return fmt.Errorf("repo bucket not found")
+			}
+
+			// save repo to cache
+			err = b.Put([]byte(repo), respBytes)
+			if err != nil {
+				return err
+			}
+
+			// fetch meta-bucket
+			mb := tx.Bucket([]byte(MetaBucket))
+			if mb == nil {
+				return fmt.Errorf("high score bucket not found")
+			}
+
+			// update total repos count
+			if isNewRepo {
+				err = updateReposCount(mb, resp, repo)
+				if err != nil {
+					return err
+				}
+			}
+
+			return updateHighScores(mb, resp, repo)
+		})
+
+		if err != nil {
+			log.Println("Bolt writing error:", err)
+		}
+	}
+
+	return
+}
+
+func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
+	// check if we need to update the high score list
+	if resp.Files < 100 {
+		// only repos with >= 100 files are considered for the high score list
+		return nil
+	}
+
+	// start updating high score list
+	scoreBytes := mb.Get([]byte("scores"))
+	if scoreBytes == nil {
+		scoreBytes, _ = json.Marshal([]scoreHeap{})
+	}
+	scores := &scoreHeap{}
+	json.Unmarshal(scoreBytes, scores)
+
+	heap.Init(scores)
+	if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 100 {
+		// lowest score on list is higher than this repo's score, so no need to add, unless
+		// we do not have 100 high scores yet
+		return nil
+	}
+	// if this repo is already in the list, remove the original entry:
+	for i := range *scores {
+		if (*scores)[i].Repo == repo {
+			heap.Remove(scores, i)
+			break
+		}
+	}
+	// now we can safely push it onto the heap
+	heap.Push(scores, scoreItem{
+		Repo:  repo,
+		Score: resp.Average * 100.0,
+		Files: resp.Files,
+	})
+	if len(*scores) > 100 {
+		// trim heap if it's grown to over 100
+		*scores = (*scores)[:100]
+	}
+	scoreBytes, err := json.Marshal(&scores)
+	if err != nil {
+		return err
+	}
+	err = mb.Put([]byte("scores"), scoreBytes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func updateReposCount(mb *bolt.Bucket, resp checksResp, repo string) (err error) {
+	log.Printf("New repo %q, adding to repo count...", repo)
+	totalInt := 0
+	total := mb.Get([]byte("total_repos"))
+	if total != nil {
+		err = json.Unmarshal(total, &totalInt)
+		if err != nil {
+			return fmt.Errorf("could not unmarshal total repos count: %v", err)
+		}
+	}
+	totalInt++ // increase repo count
+	total, err = json.Marshal(totalInt)
+	if err != nil {
+		return fmt.Errorf("could not marshal total repos count: %v", err)
+	}
+	mb.Put([]byte("total_repos"), total)
+	log.Println("Repo count is now", totalInt)
+	return nil
 }

handlers/checks.go

@@ -1,6 +1,8 @@
 package handlers
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -8,33 +10,41 @@ import (
 	"strings"
 	"time"
 
+	"github.com/boltdb/bolt"
 	"github.com/gojp/goreportcard/check"
-	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
-)
-
-var (
-	mongoURL        = "mongodb://127.0.0.1:27017"
-	mongoDatabase   = "goreportcard"
-	mongoCollection = "reports"
 )
 
 func getFromCache(repo string) (checksResp, error) {
-	// try and fetch from mongo
-	session, err := mgo.Dial(mongoURL)
+	// try and fetch from boltdb
+	db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
-		return checksResp{}, fmt.Errorf("Failed to get mongo collection during GET: %v", err)
+		return checksResp{}, fmt.Errorf("Failed to open bolt database during GET: %v", err)
 	}
-	defer session.Close()
-	coll := session.DB(mongoDatabase).C(mongoCollection)
+	defer db.Close()
+
 	resp := checksResp{}
-	err = coll.Find(bson.M{"repo": repo}).One(&resp)
+	err = db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(RepoBucket))
+		if b == nil {
+			return errors.New("No repo bucket")
+		}
+		cached := b.Get([]byte(repo))
+		if cached == nil {
+			return fmt.Errorf("%q not found in cache", repo)
+		}
+
+		err = json.Unmarshal(cached, &resp)
+		if err != nil {
+			return fmt.Errorf("failed to parse JSON for %q in cache", repo)
+		}
+		return nil
+	})
+
 	if err != nil {
-		return checksResp{}, fmt.Errorf("Failed to fetch %q from mongo: %v", repo, err)
+		return resp, err
 	}
 
 	resp.LastRefresh = resp.LastRefresh.UTC()
-
 	return resp, nil
 }
 

handlers/high_scores.go

@@ -1,14 +1,16 @@
 package handlers
 
 import (
+	"container/heap"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
 	"text/template"
+	"time"
 
+	"github.com/boltdb/bolt"
 	"github.com/dustin/go-humanize"
-	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
 )
 
 func add(x, y int) int {
@@ -16,35 +18,44 @@ func add(x, y int) int {
 }
 
 func formatScore(x float64) string {
-	return fmt.Sprintf("%.2f", x*100)
+	return fmt.Sprintf("%.2f", x)
 }
 
 // HighScoresHandler handles the stats page
 func HighScoresHandler(w http.ResponseWriter, r *http.Request) {
-	session, err := mgo.Dial(mongoURL)
+	// write to boltdb
+	db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
-		log.Println("ERROR: could not get collection:", err)
-		http.Error(w, err.Error(), 500)
+		log.Println("Failed to open bolt database: ", err)
 		return
 	}
-	defer session.Close()
-	coll := session.DB(mongoDatabase).C(mongoCollection)
+	defer db.Close()
 
-	var highScores []struct {
-		Repo    string
-		Files   int
-		Average float64
-	}
-	err = coll.Find(bson.M{"files": bson.M{"$gt": 100}}).Sort("-average").Limit(50).All(&highScores)
-	if err != nil {
-		log.Println("ERROR: could not get high scores: ", err)
-		http.Error(w, err.Error(), 500)
-		return
-	}
+	count, scores := 0, &scoreHeap{}
+	err = db.View(func(tx *bolt.Tx) error {
+		hsb := tx.Bucket([]byte(MetaBucket))
+		if hsb == nil {
+			return fmt.Errorf("High score bucket not found")
+		}
+		scoreBytes := hsb.Get([]byte("scores"))
+		if scoreBytes == nil {
+			scoreBytes, _ = json.Marshal([]scoreHeap{})
+		}
+		json.Unmarshal(scoreBytes, scores)
+
+		heap.Init(scores)
+
+		total := hsb.Get([]byte("total_repos"))
+		if total == nil {
+			count = 0
+		} else {
+			json.Unmarshal(total, &count)
+		}
+		return nil
+	})
 
-	count, err := coll.Count()
 	if err != nil {
-		log.Println("ERROR: could not get count of high scores: ", err)
+		log.Println("ERROR: Failed to load high scores from bolt database: ", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
@@ -52,5 +63,10 @@ func HighScoresHandler(w http.ResponseWriter, r *http.Request) {
 	funcs := template.FuncMap{"add": add, "formatScore": formatScore}
 	t := template.Must(template.New("high_scores.html").Funcs(funcs).ParseFiles("templates/high_scores.html"))
 
-	t.Execute(w, map[string]interface{}{"HighScores": highScores, "Count": humanize.Comma(int64(count))})
+	sortedScores := make([]scoreItem, len(*scores))
+	for i := range sortedScores {
+		sortedScores[len(sortedScores)-i-1] = heap.Pop(scores).(scoreItem)
+	}
+
+	t.Execute(w, map[string]interface{}{"HighScores": sortedScores, "Count": humanize.Comma(int64(count))})
 }

handlers/score_heap.go

@@ -0,0 +1,28 @@
+package handlers
+
+type scoreItem struct {
+	Repo  string  `json:"repo"`
+	Score float64 `json:"score"`
+	Files int     `json:"files"`
+}
+
+// An scoreHeap is a min-heap of ints.
+type scoreHeap []scoreItem
+
+func (h scoreHeap) Len() int           { return len(h) }
+func (h scoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score }
+func (h scoreHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *scoreHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	*h = append(*h, x.(scoreItem))
+}
+
+func (h *scoreHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}

main.go

@@ -6,7 +6,9 @@ import (
 	"net/http"
 	"os"
 	"regexp"
+	"time"
 
+	"github.com/boltdb/bolt"
 	"github.com/gojp/goreportcard/handlers"
 )
 
@@ -31,11 +33,36 @@ func makeHandler(name string, fn func(http.ResponseWriter, *http.Request, string
 	}
 }
 
+// initDB opens the bolt database file (or creates it if it does not exist), and creates
+// a bucket for saving the repos, also only if it does not exist.
+func initDB() error {
+	db, err := bolt.Open(handlers.DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(handlers.RepoBucket))
+		if err != nil {
+			return err
+		}
+		_, err = tx.CreateBucketIfNotExists([]byte(handlers.MetaBucket))
+		return err
+	})
+	return err
+}
+
 func main() {
 	if err := os.MkdirAll("repos/src/github.com", 0755); err != nil && !os.IsExist(err) {
 		log.Fatal("ERROR: could not create repos dir: ", err)
 	}
 
+	// initialize database
+	if err := initDB(); err != nil {
+		log.Fatal("ERROR: could not open bolt db: ", err)
+	}
+
 	http.HandleFunc("/assets/", handlers.AssetsHandler)
 	http.HandleFunc("/checks", handlers.CheckHandler)
 	http.HandleFunc("/report/", makeHandler("report", handlers.ReportHandler))

templates/high_scores.html

@@ -12,10 +12,10 @@
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
       })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
-    
+
       ga('create', 'UA-58936835-1', 'auto');
       ga('send', 'pageview');
-    
+
     </script>
     <style>
     /* Footer section!
@@ -47,7 +47,8 @@
 </head>
 <body>
   <div class="container">
-    <h1> High Scores</h1>
+    <h1>High Scores</h1>
+    <p><a href="/">Back</a></p>
     <table class="table">
       <thead>
         <tr>
@@ -63,7 +64,7 @@
       <td><a href="/report/{{ $highScore.Repo}}">{{ add $index 1 }}</td></a>
       <td><a href="https://github.com/{{ $highScore.Repo }}" rel="nofollow">{{ $highScore.Repo }}</a></td>
       <td>{{ $highScore.Files }}</td>
-      <td>{{ formatScore $highScore.Average }}</td>
+      <td>{{ formatScore $highScore.Score }}</td>
       </tr>
     {{end}}
     </tbody>
@@ -78,7 +79,7 @@
       <div class="row">
         <div class="col-md-12">
           <p class="footer-text">
-          Made by 
+          Made by
             <a href="https://twitter.com/shawnps">@shawnps</a> and
             <a href="https://twitter.com/ironzeb">@ironzeb</a>. The Go Gopher was created by <a href="http://reneefrench.blogspot.com/">Renée French</a>
           </p>

tools/mongotobolt.go

@@ -0,0 +1,207 @@
+package main
+
+import (
+	"container/heap"
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/gojp/goreportcard/check"
+	"github.com/gojp/goreportcard/handlers"
+
+	"gopkg.in/mgo.v2"
+)
+
+const (
+	dbPath     string = "goreportcard.db"
+	repoBucket string = "repos"
+	metaBucket string = "meta"
+
+	mongoURL        = "mongodb://127.0.0.1:27017"
+	mongoDatabase   = "goreportcard"
+	mongoCollection = "reports"
+)
+
+type Grade string
+
+type score struct {
+	Name          string              `json:"name"`
+	Description   string              `json:"description"`
+	FileSummaries []check.FileSummary `json:"file_summaries"`
+	Weight        float64             `json:"weight"`
+	Percentage    float64             `json:"percentage"`
+}
+
+type checksResp struct {
+	Checks      []score   `json:"checks"`
+	Average     float64   `json:"average"`
+	Grade       Grade     `json:"grade"`
+	Files       int       `json:"files"`
+	Issues      int       `json:"issues"`
+	Repo        string    `json:"repo"`
+	LastRefresh time.Time `json:"last_refresh"`
+}
+
+// initDB opens the bolt database file (or creates it if it does not exist), and creates
+// a bucket for saving the repos, also only if it does not exist.
+func initDB() error {
+	db, err := bolt.Open(handlers.DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(repoBucket))
+		if err != nil {
+			return err
+		}
+		_, err = tx.CreateBucketIfNotExists([]byte(metaBucket))
+		return err
+	})
+	return err
+}
+
+func main() {
+	// initialize bolt database
+	if err := initDB(); err != nil {
+		log.Fatal("ERROR: could not open bolt db: ", err)
+	}
+
+	session, err := mgo.Dial(mongoURL)
+	if err != nil {
+		log.Fatal("ERROR: could not get collection:", err)
+	}
+	defer session.Close()
+	coll := session.DB(mongoDatabase).C(mongoCollection)
+
+	db, err := bolt.Open(dbPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		log.Println("Failed to open bolt database: ", err)
+		return
+	}
+	defer db.Close()
+
+	var repos []checksResp
+	coll.Find(nil).All(&repos)
+
+	for _, repo := range repos {
+		fmt.Printf("inserting %q into bolt...\n", repo.Repo)
+		err = db.Update(func(tx *bolt.Tx) error {
+			bkt := tx.Bucket([]byte(repoBucket))
+			if bkt == nil {
+				return fmt.Errorf("repo bucket not found")
+			}
+			b, err := json.Marshal(repo)
+			if err != nil {
+				return err
+			}
+
+			mb := tx.Bucket([]byte(metaBucket))
+			if mb == nil {
+				return fmt.Errorf("repo bucket not found")
+			}
+			updateHighScores(mb, repo, repo.Repo)
+
+			return bkt.Put([]byte(repo.Repo), b)
+		})
+		if err != nil {
+			log.Println("Bolt writing error:", err)
+		}
+	}
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		mb := tx.Bucket([]byte(metaBucket))
+		if mb == nil {
+			return fmt.Errorf("repo bucket not found")
+		}
+		totalInt := len(repos)
+		total, err := json.Marshal(totalInt)
+		if err != nil {
+			return fmt.Errorf("could not marshal total repos count: %v", err)
+		}
+		return mb.Put([]byte("total_repos"), total)
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
+	// check if we need to update the high score list
+	if resp.Files < 100 {
+		// only repos with >= 100 files are considered for the high score list
+		return nil
+	}
+
+	// start updating high score list
+	scoreBytes := mb.Get([]byte("scores"))
+	if scoreBytes == nil {
+		scoreBytes, _ = json.Marshal([]scoreHeap{})
+	}
+	scores := &scoreHeap{}
+	json.Unmarshal(scoreBytes, scores)
+
+	heap.Init(scores)
+	if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 100 {
+		// lowest score on list is higher than this repo's score, so no need to add, unless
+		// we do not have 100 high scores yet
+		return nil
+	}
+	// if this repo is already in the list, remove the original entry:
+	for i := range *scores {
+		if (*scores)[i].Repo == repo {
+			heap.Remove(scores, i)
+			break
+		}
+	}
+	// now we can safely push it onto the heap
+	heap.Push(scores, scoreItem{
+		Repo:  repo,
+		Score: resp.Average * 100.0,
+		Files: resp.Files,
+	})
+	if len(*scores) > 100 {
+		// trim heap if it's grown to over 100
+		*scores = (*scores)[:100]
+	}
+	scoreBytes, err := json.Marshal(&scores)
+	if err != nil {
+		return err
+	}
+	err = mb.Put([]byte("scores"), scoreBytes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type scoreItem struct {
+	Repo  string  `json:"repo"`
+	Score float64 `json:"score"`
+	Files int     `json:"files"`
+}
+
+// An scoreHeap is a min-heap of ints.
+type scoreHeap []scoreItem
+
+func (h scoreHeap) Len() int           { return len(h) }
+func (h scoreHeap) Less(i, j int) bool { return h[i].Score < h[j].Score }
+func (h scoreHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *scoreHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	*h = append(*h, x.(scoreItem))
+}
+
+func (h *scoreHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}