From c716fb39d8ae2e3d3d767c1419997d2a529329a7 Mon Sep 17 00:00:00 2001
From: Steve Manuel <nilslice@gmail.com>
Date: Fri, 28 Oct 2016 17:06:04 -0700
Subject: code reorg and debugging bolt issues

---
 system/api/analytics/init.go | 103 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 system/api/analytics/init.go

(limited to 'system/api/analytics/init.go')

diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go
new file mode 100644
index 0000000..c04a7f2
--- /dev/null
+++ b/system/api/analytics/init.go
@@ -0,0 +1,103 @@
+// Package analytics provides the methods to run an analytics reporting system
+// for API requests which may be useful to users for measuring access and
+// possibly identifying bad actors abusing requests.
+package analytics
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/boltdb/bolt"
+)
+
+type apiRequest struct {
+	URL        string `json:"url"`
+	Method     string `json:"http_method"`
+	RemoteAddr string `json:"ip_address"`
+	Timestamp  int64  `json:"timestamp"`
+	External   bool   `json:"external"`
+}
+
+var (
+	store      *bolt.DB
+	recordChan chan apiRequest
+)
+
+// Record queues an apiRequest for metrics
+func Record(req *http.Request) {
+	external := strings.Contains(req.URL.Path, "/external/")
+
+	r := apiRequest{
+		URL:        req.URL.String(),
+		Method:     req.Method,
+		RemoteAddr: req.RemoteAddr,
+		Timestamp:  time.Now().Unix() * 1000,
+		External:   external,
+	}
+
+	// put r on buffered recordChan to take advantage of batch insertion in DB
+	recordChan <- r
+
+}
+
+// Close exports the abillity to close our db file. Should be called with defer
+// after call to Init() from the same place.
+func Close() {
+	err := store.Close()
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+// Init creates a db connection, should run an initial prune of old data, and
+// sets up the queue/batching channel
+func Init() {
+	store, err := bolt.Open("analytics.db", 0666, nil)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Println("analytics", store)
+
+	recordChan = make(chan apiRequest, 1024*128)
+
+	go serve()
+
+	err = store.Update(func(tx *bolt.Tx) error {
+
+		return nil
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func serve() {
+	// make timer to notify select to batch request insert from recordChan
+	// interval: 1 minute
+	apiRequestTimer := time.NewTicker(time.Minute * 1)
+
+	// make timer to notify select to remove old analytics
+	// interval: 2 weeks
+	// TODO: enable analytics backup service to cloud
+	pruneDBTimer := time.NewTicker(time.Hour * 24 * 14)
+
+	for {
+		select {
+		case <-apiRequestTimer.C:
+			var reqs []apiRequest
+			batchSize := len(recordChan)
+
+			for i := 0; i < batchSize; i++ {
+				reqs = append(reqs, <-recordChan)
+			}
+
+		case <-pruneDBTimer.C:
+
+		default:
+		}
+	}
+}
-- 
cgit v1.2.3


From b6eb040045fcfeb39dda4fcd440d2be184c8e715 Mon Sep 17 00:00:00 2001
From: Steve Manuel <nilslice@gmail.com>
Date: Sat, 29 Oct 2016 01:33:33 -0700
Subject: adding db/analytics init back in after reverting for debug

---
 system/api/analytics/init.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'system/api/analytics/init.go')

diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go
index c04a7f2..c351bed 100644
--- a/system/api/analytics/init.go
+++ b/system/api/analytics/init.go
@@ -4,7 +4,6 @@
 package analytics
 
 import (
-	"fmt"
 	"log"
 	"net/http"
 	"strings"
@@ -55,13 +54,12 @@ func Close() {
 // Init creates a db connection, should run an initial prune of old data, and
 // sets up the queue/batching channel
 func Init() {
-	store, err := bolt.Open("analytics.db", 0666, nil)
+	var err error
+	store, err = bolt.Open("analytics.db", 0666, nil)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
-	fmt.Println("analytics", store)
-
 	recordChan = make(chan apiRequest, 1024*128)
 
 	go serve()
-- 
cgit v1.2.3