From adde538e9ade90ae6baba49073b9848be8e9bc56 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 19:55:05 -0800 Subject: adding initial caching of daily metrics from api requests --- system/api/analytics/init.go | 89 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 14 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 6558adf..9a5f428 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -24,6 +24,12 @@ type apiRequest struct { External bool `json:"external"` } +type apiMetric struct { + Date string `json:"date"` + Total int `json:"total"` + Unique int `json:"unique"` +} + var ( store *bolt.DB requestChan chan apiRequest @@ -71,6 +77,11 @@ func Init() { return err } + _, err = tx.CreateBucketIfNotExists([]byte("__metrics")) + if err != nil { + return err + } + return nil }) if err != nil { @@ -92,7 +103,7 @@ func serve() { apiRequestTimer := time.NewTicker(time.Second * 30) // make timer to notify select to remove analytics older than 14 days - // interval: 1 weeks + // interval: 1 week // TODO: enable analytics backup service to cloud pruneThreshold := time.Hour * 24 * 14 pruneDBTimer := time.NewTicker(pruneThreshold / 2) @@ -119,12 +130,20 @@ func serve() { // ChartData returns the map containing decoded javascript needed to chart 2 weeks of data by day func ChartData() (map[string]interface{}, error) { - // set thresholds for today and the 6 days preceeding + // set thresholds for today and the 13 days preceeding times := [14]time.Time{} dates := [14]string{} now := time.Now() today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + ips := [14]map[string]struct{}{} + for i := range ips { + ips[i] = make(map[string]struct{}) + } + + total := [14]int{} + unique := [14]int{} + for i := range times { // subtract 24 * i hours to make days prior dur := time.Duration(24 * i * -1) @@ -135,20 +154,57 @@ func ChartData() (map[string]interface{}, error) { dates[len(times)-1-i] = day.Format("01/02") } - // get api request analytics from db + // get api request analytics and metrics from db var requests = []apiRequest{} + var metrics = [14]apiMetric{} + err := store.View(func(tx *bolt.Tx) error { + m := tx.Bucket([]byte("__metrics")) b := tx.Bucket([]byte("__requests")) - err := b.ForEach(func(k, v []byte) error { + err := m.ForEach(func(k, v []byte) error { + var metric apiMetric + err := json.Unmarshal(v, &metric) + if err != nil { + log.Println("Error decoding api metric json from analytics db:", err) + return nil + } + + // if the metric date is in current date range, insert it into + // metrics array at the position of the date in dates array + for i := range dates { + if metric.Date == dates[i] { + metrics[i] = metric + } + } + + return nil + }) + if err != nil { + return err + } + + err = b.ForEach(func(k, v []byte) error { var r apiRequest err := json.Unmarshal(v, &r) if err != nil { - log.Println("Error decoding json from analytics db:", err) + log.Println("Error decoding api request json from analytics db:", err) return nil } - requests = append(requests, r) + // delete the record in db if it belongs to a day already in metrics, + // otherwise append it to requests to be analyzed + d := time.Unix(r.Timestamp/1000, 0).Format("01/02") + for i := range dates { + if metrics[i].Date == d { + err := b.Delete(k) + if err != nil { + return err + } + } else { + requests = append(requests, r) + } + } return nil }) @@ -162,14 +218,6 @@ func ChartData() (map[string]interface{}, error) { return nil, err } - ips := [14]map[string]struct{}{} - for i := range ips { - ips[i] = make(map[string]struct{}) - } - - total := [14]int{} - unique := [14]int{} - CHECK_REQUEST: for i := range requests { ts := time.Unix(requests[i].Timestamp/1000, 0) @@ -222,6 +270,19 @@ CHECK_REQUEST: } } + // loop through total and unique to see which dates are accounted for and + // insert data from metrics array where dates are not + for i := range metrics { + if total[i] == 0 { + total[i] = metrics[i].Total + } + + if unique[i] == 0 { + unique[i] = metrics[i].Unique + } + } + + // marshal array counts to js arrays for output to chart jsUnique, err := json.Marshal(unique) if err != nil { return nil, err -- cgit v1.2.3 From f902c05ae661bad0745e8869718af4cdcc015e17 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 20:15:42 -0800 Subject: checking for metric data in db first --- system/api/analytics/init.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 9a5f428..95ad8f4 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -195,15 +195,13 @@ func ChartData() (map[string]interface{}, error) { // delete the record in db if it belongs to a day already in metrics, // otherwise append it to requests to be analyzed d := time.Unix(r.Timestamp/1000, 0).Format("01/02") - for i := range dates { - if metrics[i].Date == d { - err := b.Delete(k) - if err != nil { - return err - } - } else { - requests = append(requests, r) + if m.Get([]byte(d)) != nil { + err := b.Delete(k) + if err != nil { + return err } + } else { + requests = append(requests, r) } return nil -- cgit v1.2.3 From 0c1c118ae3e42c92cf9d47e41903a8c256019935 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 21:09:32 -0800 Subject: inserting metrics data into cache --- system/api/analytics/init.go | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 95ad8f4..1bd5507 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -158,7 +158,7 @@ func ChartData() (map[string]interface{}, error) { var requests = []apiRequest{} var metrics = [14]apiMetric{} - err := store.View(func(tx *bolt.Tx) error { + err := store.Update(func(tx *bolt.Tx) error { m := tx.Bucket([]byte("__metrics")) b := tx.Bucket([]byte("__requests")) @@ -270,14 +270,36 @@ CHECK_REQUEST: // loop through total and unique to see which dates are accounted for and // insert data from metrics array where dates are not - for i := range metrics { - if total[i] == 0 { - total[i] = metrics[i].Total - } + err = store.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("__metrics")) + + for i := range metrics { + if total[i] == 0 { + total[i] = metrics[i].Total + } + + if unique[i] == 0 { + unique[i] = metrics[i].Unique + } + + k := metrics[i].Date + if b.Get([]byte(k)) == nil { + v, err := json.Marshal(metrics[i]) + if err != nil { + return err + } - if unique[i] == 0 { - unique[i] = metrics[i].Unique + err = b.Put([]byte(k), v) + if err != nil { + return err + } + } } + + return nil + }) + if err != nil { + return nil, err } // marshal array counts to js arrays for output to chart -- cgit v1.2.3 From 92e01ee742f1f8e073fbc18b153e2adf65516cad Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 21:14:41 -0800 Subject: using dates as range for metrics put --- system/api/analytics/init.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 1bd5507..e883e3b 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -273,7 +273,7 @@ CHECK_REQUEST: err = store.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("__metrics")) - for i := range metrics { + for i := range dates { if total[i] == 0 { total[i] = metrics[i].Total } @@ -282,7 +282,7 @@ CHECK_REQUEST: unique[i] = metrics[i].Unique } - k := metrics[i].Date + k := dates[i] if b.Get([]byte(k)) == nil { v, err := json.Marshal(metrics[i]) if err != nil { -- cgit v1.2.3 From e67f85c3ba95edb05eb155408f1d650ec85c55fe Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 23:21:08 -0800 Subject: pulling data range literals out to const, adding check if metric count is 0 before writing to cache --- system/api/analytics/init.go | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index e883e3b..f7b91d2 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -21,7 +21,7 @@ type apiRequest struct { Proto string `json:"http_protocol"` RemoteAddr string `json:"ip_address"` Timestamp int64 `json:"timestamp"` - External bool `json:"external"` + External bool `json:"external_content"` } type apiMetric struct { @@ -35,6 +35,10 @@ var ( requestChan chan apiRequest ) +// RANGE determines the number of days ponzu request analytics and metrics are +// stored and displayed within the system +const RANGE = 14 + // Record queues an apiRequest for metrics func Record(req *http.Request) { external := strings.Contains(req.URL.Path, "/external/") @@ -102,10 +106,10 @@ func serve() { // interval: 30 seconds apiRequestTimer := time.NewTicker(time.Second * 30) - // make timer to notify select to remove analytics older than 14 days - // interval: 1 week + // make timer to notify select to remove analytics older than RANGE days + // interval: RANGE/2 days // TODO: enable analytics backup service to cloud - pruneThreshold := time.Hour * 24 * 14 + pruneThreshold := time.Hour * 24 * RANGE pruneDBTimer := time.NewTicker(pruneThreshold / 2) for { @@ -130,19 +134,19 @@ func serve() { // ChartData returns the map containing decoded javascript needed to chart 2 weeks of data by day func ChartData() (map[string]interface{}, error) { - // set thresholds for today and the 13 days preceeding - times := [14]time.Time{} - dates := [14]string{} + // set thresholds for today and the RANGE-1 days preceeding + times := [RANGE]time.Time{} + dates := [RANGE]string{} now := time.Now() today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - ips := [14]map[string]struct{}{} + ips := [RANGE]map[string]struct{}{} for i := range ips { ips[i] = make(map[string]struct{}) } - total := [14]int{} - unique := [14]int{} + total := [RANGE]int{} + unique := [RANGE]int{} for i := range times { // subtract 24 * i hours to make days prior @@ -156,7 +160,7 @@ func ChartData() (map[string]interface{}, error) { // get api request analytics and metrics from db var requests = []apiRequest{} - var metrics = [14]apiMetric{} + var metrics = [RANGE]apiMetric{} err := store.Update(func(tx *bolt.Tx) error { m := tx.Bucket([]byte("__metrics")) @@ -284,14 +288,16 @@ CHECK_REQUEST: k := dates[i] if b.Get([]byte(k)) == nil { - v, err := json.Marshal(metrics[i]) - if err != nil { - return err - } + if metrics[i].Total != 0 { + v, err := json.Marshal(metrics[i]) + if err != nil { + return err + } - err = b.Put([]byte(k), v) - if err != nil { - return err + err = b.Put([]byte(k), v) + if err != nil { + return err + } } } } -- cgit v1.2.3 From ff66f6e8a6b7fd5e24f9cb900ddf55e0e495c49d Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 12 Dec 2016 23:29:08 -0800 Subject: adding debug prints --- system/api/analytics/init.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index f7b91d2..01950ee 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -5,6 +5,7 @@ package analytics import ( "encoding/json" + "fmt" "log" "net/http" "runtime" @@ -182,6 +183,8 @@ func ChartData() (map[string]interface{}, error) { } } + fmt.Println(metrics) + return nil }) if err != nil { -- cgit v1.2.3 From e73f2d30b2a1a56a7aa629b6ffcd0b1d50bd91d9 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 08:25:29 -0800 Subject: removing debug prints and adding a check for data cached or needs analysis --- system/api/analytics/init.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 01950ee..0b24825 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -5,7 +5,6 @@ package analytics import ( "encoding/json" - "fmt" "log" "net/http" "runtime" @@ -183,8 +182,6 @@ func ChartData() (map[string]interface{}, error) { } } - fmt.Println(metrics) - return nil }) if err != nil { @@ -199,15 +196,10 @@ func ChartData() (map[string]interface{}, error) { return nil } - // delete the record in db if it belongs to a day already in metrics, - // otherwise append it to requests to be analyzed - d := time.Unix(r.Timestamp/1000, 0).Format("01/02") - if m.Get([]byte(d)) != nil { - err := b.Delete(k) - if err != nil { - return err - } - } else { + // append request to requests for analysis if its timestamp is today + // or its day is not already in cache + d := time.Unix(r.Timestamp/1000, 0) + if !d.Before(today) || m.Get([]byte(d.Format("01/02"))) != nil { requests = append(requests, r) } -- cgit v1.2.3 From feb10ec883258f3b7239398bbc897cec57bab339 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 08:55:50 -0800 Subject: changing or -> and op in check --- system/api/analytics/init.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 0b24825..bced50d 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -197,9 +197,9 @@ func ChartData() (map[string]interface{}, error) { } // append request to requests for analysis if its timestamp is today - // or its day is not already in cache + // and its day is not already in cache d := time.Unix(r.Timestamp/1000, 0) - if !d.Before(today) || m.Get([]byte(d.Format("01/02"))) != nil { + if !d.Before(today) && m.Get([]byte(d.Format("01/02"))) != nil { requests = append(requests, r) } @@ -281,15 +281,15 @@ CHECK_REQUEST: unique[i] = metrics[i].Unique } - k := dates[i] - if b.Get([]byte(k)) == nil { + k := []byte(dates[i]) + if b.Get(k) == nil { if metrics[i].Total != 0 { v, err := json.Marshal(metrics[i]) if err != nil { return err } - err = b.Put([]byte(k), v) + err = b.Put(k, v) if err != nil { return err } -- cgit v1.2.3 From a92afa283947f1bbf9b12525982af74f6ce1f469 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 09:02:07 -0800 Subject: flipping comparison to match previous op change --- system/api/analytics/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index bced50d..9942dc2 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -199,7 +199,7 @@ func ChartData() (map[string]interface{}, error) { // append request to requests for analysis if its timestamp is today // and its day is not already in cache d := time.Unix(r.Timestamp/1000, 0) - if !d.Before(today) && m.Get([]byte(d.Format("01/02"))) != nil { + if !d.Before(today) && m.Get([]byte(d.Format("01/02"))) == nil { requests = append(requests, r) } -- cgit v1.2.3 From c0a2d56bb102b52f06a87851fc8a7164ef306358 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 09:13:45 -0800 Subject: testing memory cache of current metrics rather than lookup in db --- system/api/analytics/init.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 9942dc2..fc7ecea 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -161,6 +161,7 @@ func ChartData() (map[string]interface{}, error) { // get api request analytics and metrics from db var requests = []apiRequest{} var metrics = [RANGE]apiMetric{} + currentMetrics := make(map[string]struct{}) err := store.Update(func(tx *bolt.Tx) error { m := tx.Bucket([]byte("__metrics")) @@ -174,6 +175,9 @@ func ChartData() (map[string]interface{}, error) { return nil } + // add metric to currentMetrics map + currentMetrics[metric.Date] = struct{}{} + // if the metric date is in current date range, insert it into // metrics array at the position of the date in dates array for i := range dates { @@ -199,7 +203,8 @@ func ChartData() (map[string]interface{}, error) { // append request to requests for analysis if its timestamp is today // and its day is not already in cache d := time.Unix(r.Timestamp/1000, 0) - if !d.Before(today) && m.Get([]byte(d.Format("01/02"))) == nil { + _, inCache := currentMetrics[d.Format("01/02")] + if !d.Before(today) && !inCache { requests = append(requests, r) } -- cgit v1.2.3 From 1cae6f9095e0477becb072d0d2a8cf956e955f56 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 10:17:33 -0800 Subject: adding fix for metrics storage and caching --- system/api/analytics/init.go | 64 ++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 26 deletions(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index fc7ecea..6c66007 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -160,8 +160,7 @@ func ChartData() (map[string]interface{}, error) { // get api request analytics and metrics from db var requests = []apiRequest{} - var metrics = [RANGE]apiMetric{} - currentMetrics := make(map[string]struct{}) + currentMetrics := make(map[string]apiMetric) err := store.Update(func(tx *bolt.Tx) error { m := tx.Bucket([]byte("__metrics")) @@ -176,15 +175,7 @@ func ChartData() (map[string]interface{}, error) { } // add metric to currentMetrics map - currentMetrics[metric.Date] = struct{}{} - - // if the metric date is in current date range, insert it into - // metrics array at the position of the date in dates array - for i := range dates { - if metric.Date == dates[i] { - metrics[i] = metric - } - } + currentMetrics[metric.Date] = metric return nil }) @@ -201,10 +192,10 @@ func ChartData() (map[string]interface{}, error) { } // append request to requests for analysis if its timestamp is today - // and its day is not already in cache + // or if its day is not already in cache d := time.Unix(r.Timestamp/1000, 0) _, inCache := currentMetrics[d.Format("01/02")] - if !d.Before(today) && !inCache { + if !d.Before(today) || !inCache { requests = append(requests, r) } @@ -272,31 +263,52 @@ CHECK_REQUEST: } } + // add data to currentMetrics from total and unique + for i := range dates { + _, ok := currentMetrics[dates[i]] + if !ok { + m := apiMetric{ + Date: dates[i], + Total: total[i], + Unique: unique[i], + } + + currentMetrics[dates[i]] = m + } + } + // loop through total and unique to see which dates are accounted for and // insert data from metrics array where dates are not err = store.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("__metrics")) for i := range dates { + // populate total and unique with cached data if needed if total[i] == 0 { - total[i] = metrics[i].Total + total[i] = currentMetrics[dates[i]].Total } if unique[i] == 0 { - unique[i] = metrics[i].Unique + unique[i] = currentMetrics[dates[i]].Unique } - k := []byte(dates[i]) - if b.Get(k) == nil { - if metrics[i].Total != 0 { - v, err := json.Marshal(metrics[i]) - if err != nil { - return err - } - - err = b.Put(k, v) - if err != nil { - return err + // check if we need to insert old data into cache - as long as it + // is not today's data + if dates[i] != today.Format("01/02") { + k := []byte(dates[i]) + if b.Get(k) == nil { + // keep zero counts out of cache in case data is added from + // other sources + if currentMetrics[dates[i]].Total != 0 { + v, err := json.Marshal(currentMetrics[dates[i]]) + if err != nil { + return err + } + + err = b.Put(k, v) + if err != nil { + return err + } } } } -- cgit v1.2.3 From b943dd6b08d00784589e34c25740c816b2d5ffb7 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 13 Dec 2016 10:26:37 -0800 Subject: deleting old records after storing in metrics as a cache --- system/api/analytics/init.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'system/api') diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 6c66007..8117f2a 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -192,11 +192,16 @@ func ChartData() (map[string]interface{}, error) { } // append request to requests for analysis if its timestamp is today - // or if its day is not already in cache + // or if its day is not already in cache, otherwise delete it d := time.Unix(r.Timestamp/1000, 0) _, inCache := currentMetrics[d.Format("01/02")] if !d.Before(today) || !inCache { requests = append(requests, r) + } else { + err := b.Delete(k) + if err != nil { + return err + } } return nil -- cgit v1.2.3