From f2fc4db1ea0dbf4a1f75fb3e52b64e6f529e5e44 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 6 Dec 2016 17:53:57 -0800 Subject: adding cache (client-based, Etag) to API responses in addition to static files uploaded --- system/admin/cache.go | 28 ---------------------------- system/admin/server.go | 5 +++-- system/api/server.go | 10 ++++++---- system/db/config.go | 11 ----------- system/db/content.go | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 45 deletions(-) delete mode 100644 system/admin/cache.go (limited to 'system') diff --git a/system/admin/cache.go b/system/admin/cache.go deleted file mode 100644 index 2abdb26..0000000 --- a/system/admin/cache.go +++ /dev/null @@ -1,28 +0,0 @@ -package admin - -import ( - "fmt" - "net/http" - "strings" - - "github.com/bosssauce/ponzu/system/db" -) - -// CacheControl sets the default cache policy on static asset responses -func CacheControl(next http.Handler) http.Handler { - return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - etag := db.ConfigCache("etag") - policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30) - res.Header().Add("Etag", etag) - res.Header().Add("Cache-Control", policy) - - if match := res.Header().Get("If-None-Match"); match != "" { - if strings.Contains(match, etag) { - res.WriteHeader(http.StatusNotModified) - return - } - } - - next.ServeHTTP(res, req) - }) -} diff --git a/system/admin/server.go b/system/admin/server.go index f80a750..3568818 100644 --- a/system/admin/server.go +++ b/system/admin/server.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/bosssauce/ponzu/system/admin/user" + "github.com/bosssauce/ponzu/system/db" ) // Run adds Handlers to default http listener for Admin @@ -41,11 +42,11 @@ func Run() { } staticDir := filepath.Join(pwd, "cmd", "ponzu", "vendor", "github.com", "bosssauce", "ponzu", "system") - http.Handle("/admin/static/", CacheControl(http.FileServer(restrict(http.Dir(staticDir))))) + http.Handle("/admin/static/", db.CacheControl(http.FileServer(restrict(http.Dir(staticDir))))) // API path needs to be registered within server package so that it is handled // even if the API server is not running. Otherwise, images/files uploaded // through the editor will not load within the admin system. uploadsDir := filepath.Join(pwd, "uploads") - http.Handle("/api/uploads/", CacheControl(http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir)))))) + http.Handle("/api/uploads/", db.CacheControl(http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir)))))) } diff --git a/system/api/server.go b/system/api/server.go index 823ec16..41d652a 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -2,15 +2,17 @@ package api import ( "net/http" + + "github.com/bosssauce/ponzu/system/db" ) // Run adds Handlers to default http listener for API func Run() { - http.HandleFunc("/api/types", CORS(Record(typesHandler))) + http.HandleFunc("/api/types", db.CacheControl(CORS(Record(typesHandler)))) - http.HandleFunc("/api/contents", CORS(Record(contentsHandler))) + http.HandleFunc("/api/contents", db.CacheControl(CORS(Record(contentsHandler)))) - http.HandleFunc("/api/content", CORS(Record(contentHandler))) + http.HandleFunc("/api/content", db.CacheControl(CORS(Record(contentHandler)))) - http.HandleFunc("/api/content/external", CORS(Record(externalContentHandler))) + http.HandleFunc("/api/content/external", db.CacheControl(CORS(Record(externalContentHandler)))) } diff --git a/system/db/config.go b/system/db/config.go index ab1c720..b5a07e4 100644 --- a/system/db/config.go +++ b/system/db/config.go @@ -2,12 +2,9 @@ package db import ( "bytes" - "encoding/base64" "encoding/json" - "fmt" "net/url" "strings" - "time" "github.com/bosssauce/ponzu/system/admin/config" @@ -124,11 +121,3 @@ func ConfigAll() ([]byte, error) { func ConfigCache(key string) string { return configCache.Get(key) } - -// NewEtag generates a new Etag for response caching -func NewEtag() string { - now := fmt.Sprintf("%d", time.Now().Unix()) - etag := base64.StdEncoding.EncodeToString([]byte(now)) - - return etag -} diff --git a/system/db/content.go b/system/db/content.go index 74a77ec..19c31d7 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -77,6 +77,12 @@ func update(ns, id string, data url.Values) (int, error) { go SortContent(ns) } + // update changes data, so invalidate client caching + err = InvalidateCache() + if err != nil { + return 0, err + } + return cid, nil } @@ -132,6 +138,12 @@ func insert(ns string, data url.Values) (int, error) { go SortContent(ns) } + // insert changes data, so invalidate client caching + err = InvalidateCache() + if err != nil { + return 0, err + } + return effectedID, nil } @@ -149,6 +161,12 @@ func DeleteContent(target string) error { return err } + // delete changes data, so invalidate client caching + err = InvalidateCache() + if err != nil { + return err + } + // exception to typical "run in goroutine" pattern: // we want to have an updated admin view as soon as this is deleted, so // in some cases, the delete and redirect is faster than the sort, -- cgit v1.2.3 From 70a841287e6f740b09ac29d2f6266281509a3163 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 6 Dec 2016 17:56:39 -0800 Subject: tracking new file --- system/db/cache.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 system/db/cache.go (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go new file mode 100644 index 0000000..93cd3a6 --- /dev/null +++ b/system/db/cache.go @@ -0,0 +1,75 @@ +package db + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +// CacheControl sets the default cache policy on static asset responses +func CacheControl(next http.Handler) http.HandlerFunc { + return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + etag := ConfigCache("etag") + policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30) + res.Header().Add("Etag", etag) + res.Header().Add("Cache-Control", policy) + + if match := res.Header().Get("If-None-Match"); match != "" { + if strings.Contains(match, etag) { + res.WriteHeader(http.StatusNotModified) + return + } + } + + next.ServeHTTP(res, req) + }) +} + +// NewEtag generates a new Etag for response caching +func NewEtag() string { + now := fmt.Sprintf("%d", time.Now().Unix()) + etag := base64.StdEncoding.EncodeToString([]byte(now)) + + return etag +} + +// InvalidateCache sets a new Etag for http responses +func InvalidateCache() error { + kv := make(map[string]interface{}) + + c, err := ConfigAll() + if err != nil { + return err + } + + err = json.Unmarshal(c, &kv) + if err != nil { + return err + } + + kv["etag"] = NewEtag() + + var data url.Values + for k, v := range kv { + switch v.(type) { + case string: + data.Set(k, v.(string)) + case []string: + vv := v.([]string) + for i := range vv { + data.Add(k, vv[i]) + } + } + } + + err = SetConfig(data) + if err != nil { + + } + + return nil +} -- cgit v1.2.3 From 09e67cdd3702a2f474b6387f80bec4fb19f74597 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 6 Dec 2016 17:59:42 -0800 Subject: checking if first pass through kv of config to call Set before Add --- system/db/cache.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go index 93cd3a6..6fefd0a 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -61,7 +61,11 @@ func InvalidateCache() error { case []string: vv := v.([]string) for i := range vv { - data.Add(k, vv[i]) + if i == 0 { + data.Set(k, vv[i]) + } else { + data.Add(k, vv[i]) + } } } } -- cgit v1.2.3 From 2cf6e8d011f61ba1d6386620ba1e0f6c5a097190 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 6 Dec 2016 18:05:10 -0800 Subject: initialize url.Values (map) prior to Add/Set --- system/db/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go index 6fefd0a..6ba00ad 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -53,7 +53,7 @@ func InvalidateCache() error { kv["etag"] = NewEtag() - var data url.Values + data := make(url.Values) for k, v := range kv { switch v.(type) { case string: -- cgit v1.2.3 From c99dcd123e886e7c62c913074c5c04ae3acc3272 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 6 Dec 2016 18:17:24 -0800 Subject: re-order middleware so Record is called even if cache is active --- system/admin/server.go | 3 ++- system/api/server.go | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'system') diff --git a/system/admin/server.go b/system/admin/server.go index 3568818..bc6f4d6 100644 --- a/system/admin/server.go +++ b/system/admin/server.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/bosssauce/ponzu/system/admin/user" + "github.com/bosssauce/ponzu/system/api" "github.com/bosssauce/ponzu/system/db" ) @@ -48,5 +49,5 @@ func Run() { // even if the API server is not running. Otherwise, images/files uploaded // through the editor will not load within the admin system. uploadsDir := filepath.Join(pwd, "uploads") - http.Handle("/api/uploads/", db.CacheControl(http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir)))))) + http.Handle("/api/uploads/", api.Record(db.CacheControl(http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir))))))) } diff --git a/system/api/server.go b/system/api/server.go index 41d652a..703a906 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -8,11 +8,11 @@ import ( // Run adds Handlers to default http listener for API func Run() { - http.HandleFunc("/api/types", db.CacheControl(CORS(Record(typesHandler)))) + http.HandleFunc("/api/types", Record(db.CacheControl(CORS(typesHandler)))) - http.HandleFunc("/api/contents", db.CacheControl(CORS(Record(contentsHandler)))) + http.HandleFunc("/api/contents", Record(db.CacheControl(CORS(contentsHandler)))) - http.HandleFunc("/api/content", db.CacheControl(CORS(Record(contentHandler)))) + http.HandleFunc("/api/content", Record(db.CacheControl(CORS(contentHandler)))) - http.HandleFunc("/api/content/external", db.CacheControl(CORS(Record(externalContentHandler)))) + http.HandleFunc("/api/content/external", Record(db.CacheControl(CORS(externalContentHandler)))) } -- cgit v1.2.3 From fee285453459ac33f53dc66052bffa7a83aca778 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Wed, 7 Dec 2016 09:27:41 -0800 Subject: adding debug prints -- cache control not effecting json responses --- system/db/cache.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go index 6ba00ad..4076848 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -20,9 +20,12 @@ func CacheControl(next http.Handler) http.HandlerFunc { if match := res.Header().Get("If-None-Match"); match != "" { if strings.Contains(match, etag) { + fmt.Println("matched etag") res.WriteHeader(http.StatusNotModified) return } + + fmt.Println("checked, no match") } next.ServeHTTP(res, req) -- cgit v1.2.3 From a20025b452934069fc6c39949d7b9e123a503c9c Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Wed, 7 Dec 2016 09:32:46 -0800 Subject: testing cache control exec from CORS --- system/api/handlers.go | 4 ++-- system/api/server.go | 14 +++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) (limited to 'system') diff --git a/system/api/handlers.go b/system/api/handlers.go index 8a1517b..c238ca9 100644 --- a/system/api/handlers.go +++ b/system/api/handlers.go @@ -200,14 +200,14 @@ func SendJSON(res http.ResponseWriter, j map[string]interface{}) { // CORS wraps a HandleFunc to response to OPTIONS requests properly func CORS(next http.HandlerFunc) http.HandlerFunc { - return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + return db.CacheControl(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { if req.Method == http.MethodOptions { SendPreflight(res) return } next.ServeHTTP(res, req) - }) + })) } // Record wraps a HandleFunc to record API requests for analytical purposes diff --git a/system/api/server.go b/system/api/server.go index 703a906..f8c9ba9 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -1,18 +1,14 @@ package api -import ( - "net/http" - - "github.com/bosssauce/ponzu/system/db" -) +import "net/http" // Run adds Handlers to default http listener for API func Run() { - http.HandleFunc("/api/types", Record(db.CacheControl(CORS(typesHandler)))) + http.HandleFunc("/api/types", Record((CORS(typesHandler)))) - http.HandleFunc("/api/contents", Record(db.CacheControl(CORS(contentsHandler)))) + http.HandleFunc("/api/contents", Record((CORS(contentsHandler)))) - http.HandleFunc("/api/content", Record(db.CacheControl(CORS(contentHandler)))) + http.HandleFunc("/api/content", Record((CORS(contentHandler)))) - http.HandleFunc("/api/content/external", Record(db.CacheControl(CORS(externalContentHandler)))) + http.HandleFunc("/api/content/external", Record((CORS(externalContentHandler)))) } -- cgit v1.2.3 From 7e06c63410304767bc6b0dce4e1e2477934fd816 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Sun, 11 Dec 2016 19:58:47 -0800 Subject: returning api server middleware to original state and adding req header lookup for cache implementation --- system/api/server.go | 12 +++++++----- system/db/cache.go | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'system') diff --git a/system/api/server.go b/system/api/server.go index f8c9ba9..823ec16 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -1,14 +1,16 @@ package api -import "net/http" +import ( + "net/http" +) // Run adds Handlers to default http listener for API func Run() { - http.HandleFunc("/api/types", Record((CORS(typesHandler)))) + http.HandleFunc("/api/types", CORS(Record(typesHandler))) - http.HandleFunc("/api/contents", Record((CORS(contentsHandler)))) + http.HandleFunc("/api/contents", CORS(Record(contentsHandler))) - http.HandleFunc("/api/content", Record((CORS(contentHandler)))) + http.HandleFunc("/api/content", CORS(Record(contentHandler))) - http.HandleFunc("/api/content/external", Record((CORS(externalContentHandler)))) + http.HandleFunc("/api/content/external", CORS(Record(externalContentHandler))) } diff --git a/system/db/cache.go b/system/db/cache.go index 4076848..e964906 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -18,7 +18,7 @@ func CacheControl(next http.Handler) http.HandlerFunc { res.Header().Add("Etag", etag) res.Header().Add("Cache-Control", policy) - if match := res.Header().Get("If-None-Match"); match != "" { + if match := req.Header.Get("If-None-Match"); match != "" { if strings.Contains(match, etag) { fmt.Println("matched etag") res.WriteHeader(http.StatusNotModified) -- cgit v1.2.3 From 3fa96eea557c4951d473f301d5172c5d7a870376 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Sun, 11 Dec 2016 20:00:50 -0800 Subject: removing print debug for cache check --- system/db/cache.go | 3 --- 1 file changed, 3 deletions(-) (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go index e964906..bfca709 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -20,12 +20,9 @@ func CacheControl(next http.Handler) http.HandlerFunc { if match := req.Header.Get("If-None-Match"); match != "" { if strings.Contains(match, etag) { - fmt.Println("matched etag") res.WriteHeader(http.StatusNotModified) return } - - fmt.Println("checked, no match") } next.ServeHTTP(res, req) -- cgit v1.2.3 From c1111448053df9ac03835b66af9c1499783def2a Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Sun, 11 Dec 2016 20:32:34 -0800 Subject: updating cache control header to be more lenient --- system/db/cache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'system') diff --git a/system/db/cache.go b/system/db/cache.go index bfca709..5f2dd03 100644 --- a/system/db/cache.go +++ b/system/db/cache.go @@ -14,8 +14,8 @@ import ( func CacheControl(next http.Handler) http.HandlerFunc { return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { etag := ConfigCache("etag") - policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30) - res.Header().Add("Etag", etag) + policy := fmt.Sprintf("max-age=%d, public", 60*60*24*30) + res.Header().Add("ETag", etag) res.Header().Add("Cache-Control", policy) if match := req.Header.Get("If-None-Match"); match != "" { -- cgit v1.2.3 From 3b10b345045428b9011eecd0ded9c04db42bf28f Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Sun, 11 Dec 2016 22:46:47 -0800 Subject: fix leftover db specifier name changes --- system/api/analytics/batch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'system') diff --git a/system/api/analytics/batch.go b/system/api/analytics/batch.go index c4a60db..68ffd65 100644 --- a/system/api/analytics/batch.go +++ b/system/api/analytics/batch.go @@ -19,7 +19,7 @@ func batchInsert(requests chan apiRequest) error { } err := store.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("requests")) + b, err := tx.CreateBucketIfNotExists([]byte("__requests")) if err != nil { return err } @@ -64,7 +64,7 @@ func batchPrune(threshold time.Duration) error { // iterate through all request data err := store.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("requests")) + b := tx.Bucket([]byte("__requests")) err := b.ForEach(func(k, v []byte) error { var r apiRequest -- cgit v1.2.3