From c716fb39d8ae2e3d3d767c1419997d2a529329a7 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Fri, 28 Oct 2016 17:06:04 -0700 Subject: code reorg and debugging bolt issues --- system/api/analytics/analytics.go | 97 ----------------------------------- system/api/analytics/init.go | 103 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 97 deletions(-) delete mode 100644 system/api/analytics/analytics.go create mode 100644 system/api/analytics/init.go (limited to 'system/api/analytics') diff --git a/system/api/analytics/analytics.go b/system/api/analytics/analytics.go deleted file mode 100644 index c30ce26..0000000 --- a/system/api/analytics/analytics.go +++ /dev/null @@ -1,97 +0,0 @@ -// Package analytics provides the methods to run an analytics reporting system -// for API requests which may be useful to users for measuring access and -// possibly identifying bad actors abusing requests. -package analytics - -import ( - "log" - "net/http" - "strings" - "time" - - "github.com/boltdb/bolt" -) - -type apiRequest struct { - URL string `json:"url"` - Method string `json:"http_method"` - RemoteAddr string `json:"ip_address"` - Timestamp int64 `json:"timestamp"` - External bool `json:"external"` -} - -var ( - store *bolt.DB - recordChan chan apiRequest -) - -// Record queues an apiRequest for metrics -func Record(req *http.Request) { - external := strings.Contains(req.URL.Path, "/external/") - - r := apiRequest{ - URL: req.URL.String(), - Method: req.Method, - RemoteAddr: req.RemoteAddr, - Timestamp: time.Now().Unix() * 1000, - External: external, - } - - // put r on buffered recordChan to take advantage of batch insertion in DB - recordChan <- r - -} - -// Close exports the abillity to close our db file. Should be called with defer -// after call to Init() from the same place. -func Close() { - err := store.Close() - if err != nil { - log.Println(err) - } -} - -// Init creates a db connection, should run an initial prune of old data, and -// sets up the queue/batching channel -func Init() { - store, err := bolt.Open("analytics.db", 0666, nil) - if err != nil { - log.Fatalln(err) - } - - recordChan = make(chan apiRequest, 1024*128) - - go serve() - - err = store.Update(func(tx *bolt.Tx) error { - - return nil - }) -} - -func serve() { - // make timer to notify select to batch request insert from recordChan - // interval: 1 minute - apiRequestTimer := time.NewTicker(time.Minute * 1) - - // make timer to notify select to remove old analytics - // interval: 2 weeks - // TODO: enable analytics backup service to cloud - pruneDBTimer := time.NewTicker(time.Hour * 24 * 14) - - for { - select { - case <-apiRequestTimer.C: - var reqs []apiRequest - batchSize := len(recordChan) - - for i := 0; i < batchSize; i++ { - reqs = append(reqs, <-recordChan) - } - - case <-pruneDBTimer.C: - - default: - } - } -} diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go new file mode 100644 index 0000000..c04a7f2 --- /dev/null +++ b/system/api/analytics/init.go @@ -0,0 +1,103 @@ +// Package analytics provides the methods to run an analytics reporting system +// for API requests which may be useful to users for measuring access and +// possibly identifying bad actors abusing requests. +package analytics + +import ( + "fmt" + "log" + "net/http" + "strings" + "time" + + "github.com/boltdb/bolt" +) + +type apiRequest struct { + URL string `json:"url"` + Method string `json:"http_method"` + RemoteAddr string `json:"ip_address"` + Timestamp int64 `json:"timestamp"` + External bool `json:"external"` +} + +var ( + store *bolt.DB + recordChan chan apiRequest +) + +// Record queues an apiRequest for metrics +func Record(req *http.Request) { + external := strings.Contains(req.URL.Path, "/external/") + + r := apiRequest{ + URL: req.URL.String(), + Method: req.Method, + RemoteAddr: req.RemoteAddr, + Timestamp: time.Now().Unix() * 1000, + External: external, + } + + // put r on buffered recordChan to take advantage of batch insertion in DB + recordChan <- r + +} + +// Close exports the abillity to close our db file. Should be called with defer +// after call to Init() from the same place. +func Close() { + err := store.Close() + if err != nil { + log.Println(err) + } +} + +// Init creates a db connection, should run an initial prune of old data, and +// sets up the queue/batching channel +func Init() { + store, err := bolt.Open("analytics.db", 0666, nil) + if err != nil { + log.Fatalln(err) + } + + fmt.Println("analytics", store) + + recordChan = make(chan apiRequest, 1024*128) + + go serve() + + err = store.Update(func(tx *bolt.Tx) error { + + return nil + }) + if err != nil { + log.Fatalln(err) + } +} + +func serve() { + // make timer to notify select to batch request insert from recordChan + // interval: 1 minute + apiRequestTimer := time.NewTicker(time.Minute * 1) + + // make timer to notify select to remove old analytics + // interval: 2 weeks + // TODO: enable analytics backup service to cloud + pruneDBTimer := time.NewTicker(time.Hour * 24 * 14) + + for { + select { + case <-apiRequestTimer.C: + var reqs []apiRequest + batchSize := len(recordChan) + + for i := 0; i < batchSize; i++ { + reqs = append(reqs, <-recordChan) + } + + case <-pruneDBTimer.C: + + default: + } + } +} -- cgit v1.2.3