From 1613413ecc3a88b2263c6ee31faa86ed615483d7 Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Fri, 24 Feb 2017 23:56:00 -0600 Subject: adding update to api --- system/api/server.go | 2 + system/api/update.go | 222 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 system/api/update.go (limited to 'system') diff --git a/system/api/server.go b/system/api/server.go index c5c1a23..6a848dd 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -9,4 +9,6 @@ func Run() { http.HandleFunc("/api/content", Record(CORS(Gzip(contentHandler)))) http.HandleFunc("/api/content/external", Record(CORS(externalContentHandler))) + + http.HandleFunc("/api/content/update", Record(CORS(updateContentHandler))) } diff --git a/system/api/update.go b/system/api/update.go new file mode 100644 index 0000000..93904dc --- /dev/null +++ b/system/api/update.go @@ -0,0 +1,222 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + "time" + + "github.com/ponzu-cms/ponzu/system/admin/upload" + "github.com/ponzu-cms/ponzu/system/admin/user" + "github.com/ponzu-cms/ponzu/system/db" + "github.com/ponzu-cms/ponzu/system/item" +) + +// Updateable accepts or rejects update POST requests to endpoints such as: +// /api/content/update?type=Review&id=1 +type Updateable interface { + // AcceptUpdate allows external content update submissions of a specific type + AcceptUpdate(http.ResponseWriter, *http.Request) error +} + +func updateContentHandler(res http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + res.WriteHeader(http.StatusMethodNotAllowed) + return + } + + err := req.ParseMultipartForm(1024 * 1024 * 4) // maxMemory 4MB + if err != nil { + log.Println("[Update] error:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + t := req.URL.Query().Get("type") + if t == "" { + res.WriteHeader(http.StatusBadRequest) + return + } + + p, found := item.Types[t] + if !found { + log.Println("[Update] attempt to submit unknown type:", t, "from:", req.RemoteAddr) + res.WriteHeader(http.StatusNotFound) + return + } + + id := req.URL.Query().Get("id") + if id == "" { + log.Println("[Update] attempt to submit update with missing id from:", req.RemoteAddr) + res.WriteHeader(http.StatusBadRequest) + return + } + + if user.IsValid(req) == false { + res.WriteHeader(http.StatusBadRequest) + return + } + + post := p() + + ext, ok := post.(Updateable) + if !ok { + log.Println("[Update] rejected non-replaceable type:", t, "from:", req.RemoteAddr) + res.WriteHeader(http.StatusBadRequest) + return + } + + ts := fmt.Sprintf("%d", int64(time.Nanosecond)*time.Now().UnixNano()/int64(time.Millisecond)) + req.PostForm.Set("timestamp", ts) + req.PostForm.Set("updated", ts) + + urlPaths, err := upload.StoreFiles(req) + if err != nil { + log.Println(err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + for name, urlPath := range urlPaths { + req.PostForm.Set(name, urlPath) + } + + // check for any multi-value fields (ex. checkbox fields) + // and correctly format for db storage. Essentially, we need + // fieldX.0: value1, fieldX.1: value2 => fieldX: []string{value1, value2} + fieldOrderValue := make(map[string]map[string][]string) + ordVal := make(map[string][]string) + for k, v := range req.PostForm { + if strings.Contains(k, ".") { + fo := strings.Split(k, ".") + + // put the order and the field value into map + field := string(fo[0]) + order := string(fo[1]) + fieldOrderValue[field] = ordVal + + // orderValue is 0:[?type=Thing&id=1] + orderValue := fieldOrderValue[field] + orderValue[order] = v + fieldOrderValue[field] = orderValue + + // discard the post form value with name.N + req.PostForm.Del(k) + } + + } + + // add/set the key & value to the post form in order + for f, ov := range fieldOrderValue { + for i := 0; i < len(ov); i++ { + position := fmt.Sprintf("%d", i) + fieldValue := ov[position] + + if req.PostForm.Get(f) == "" { + for i, fv := range fieldValue { + if i == 0 { + req.PostForm.Set(f, fv) + } else { + req.PostForm.Add(f, fv) + } + } + } else { + for _, fv := range fieldValue { + req.PostForm.Add(f, fv) + } + } + } + } + + hook, ok := post.(item.Hookable) + if !ok { + log.Println("[Update] error: Type", t, "does not implement item.Hookable or embed item.Item.") + res.WriteHeader(http.StatusBadRequest) + return + } + + err = hook.BeforeAccept(res, req) + if err != nil { + log.Println("[Update] error calling BeforeAccept:", err) + return + } + + err = ext.AcceptUpdate(res, req) + if err != nil { + log.Println("[Update] error calling Accept:", err) + return + } + + err = hook.BeforeSave(res, req) + if err != nil { + log.Println("[Update] error calling BeforeSave:", err) + return + } + + // set specifier for db bucket in case content is/isn't Trustable + var spec string + + _, err = db.SetContent(t+spec+":"+id, req.PostForm) + if err != nil { + log.Println("[Update] error calling SetContent:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + // set the target in the context so user can get saved value from db in hook + ctx := context.WithValue(req.Context(), "target", fmt.Sprintf("%s:%d", t, id)) + req = req.WithContext(ctx) + + err = hook.AfterSave(res, req) + if err != nil { + log.Println("[Update] error calling AfterSave:", err) + return + } + + err = hook.AfterAccept(res, req) + if err != nil { + log.Println("[Update] error calling AfterAccept:", err) + return + } + + // create JSON response to send data back to client + var data map[string]interface{} + if spec != "" { + spec = strings.TrimPrefix(spec, "__") + data = map[string]interface{}{ + "status": spec, + "type": t, + } + } else { + spec = "public" + data = map[string]interface{}{ + "id": id, + "status": spec, + "type": t, + } + } + + resp := map[string]interface{}{ + "data": []map[string]interface{}{ + data, + }, + } + + j, err := json.Marshal(resp) + if err != nil { + log.Println("[Update] error marshalling response to JSON:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + res.Header().Set("Content-Type", "application/json") + _, err = res.Write(j) + if err != nil { + log.Println("[Update] error writing response:", err) + return + } + +} -- cgit v1.2.3 From a7db36aa56c1998bc56d593a38f2a1640a9298a0 Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Sat, 25 Feb 2017 21:22:09 -0600 Subject: added logging for failed user validation on api update --- system/api/update.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'system') diff --git a/system/api/update.go b/system/api/update.go index 93904dc..a669a9a 100644 --- a/system/api/update.go +++ b/system/api/update.go @@ -56,6 +56,7 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { } if user.IsValid(req) == false { + log.Println("[Update] invalid user.") res.WriteHeader(http.StatusBadRequest) return } @@ -64,7 +65,7 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { ext, ok := post.(Updateable) if !ok { - log.Println("[Update] rejected non-replaceable type:", t, "from:", req.RemoteAddr) + log.Println("[Update] rejected non-updateable type:", t, "from:", req.RemoteAddr) res.WriteHeader(http.StatusBadRequest) return } -- cgit v1.2.3 From 6358ca512c227b8f035401b57543e0190c635e8d Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Thu, 2 Mar 2017 12:48:17 -0600 Subject: added UpdateContent for merge behavior on api updates, extended Hookable --- system/api/update.go | 28 ++++++++-------- system/db/content.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++ system/item/item.go | 13 ++++++++ 3 files changed, 119 insertions(+), 14 deletions(-) (limited to 'system') diff --git a/system/api/update.go b/system/api/update.go index a669a9a..00933fb 100644 --- a/system/api/update.go +++ b/system/api/update.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "errors" "fmt" "log" "net/http" @@ -10,15 +11,17 @@ import ( "time" "github.com/ponzu-cms/ponzu/system/admin/upload" - "github.com/ponzu-cms/ponzu/system/admin/user" "github.com/ponzu-cms/ponzu/system/db" "github.com/ponzu-cms/ponzu/system/item" ) +var ErrNoAuth = errors.New("Auth failed for update request.") + // Updateable accepts or rejects update POST requests to endpoints such as: // /api/content/update?type=Review&id=1 type Updateable interface { // AcceptUpdate allows external content update submissions of a specific type + // user.IsValid(req) may be checked in AcceptUpdate to validate the request AcceptUpdate(http.ResponseWriter, *http.Request) error } @@ -55,12 +58,6 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { return } - if user.IsValid(req) == false { - log.Println("[Update] invalid user.") - res.WriteHeader(http.StatusBadRequest) - return - } - post := p() ext, ok := post.(Updateable) @@ -139,15 +136,18 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { return } - err = hook.BeforeAccept(res, req) + err = hook.BeforeAcceptUpdate(res, req) if err != nil { - log.Println("[Update] error calling BeforeAccept:", err) + log.Println("[Update] error calling BeforeAcceptUpdate:", err) return } err = ext.AcceptUpdate(res, req) if err != nil { - log.Println("[Update] error calling Accept:", err) + log.Println("[Update] error calling AcceptUpdate:", err) + if err == ErrNoAuth { + res.WriteHeader(http.StatusUnauthorized) + } return } @@ -160,9 +160,9 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { // set specifier for db bucket in case content is/isn't Trustable var spec string - _, err = db.SetContent(t+spec+":"+id, req.PostForm) + _, err = db.UpdateContent(t+spec+":"+id, req.PostForm) if err != nil { - log.Println("[Update] error calling SetContent:", err) + log.Println("[Update] error calling UpdateContent:", err) res.WriteHeader(http.StatusInternalServerError) return } @@ -177,9 +177,9 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { return } - err = hook.AfterAccept(res, req) + err = hook.AfterAcceptUpdate(res, req) if err != nil { - log.Println("[Update] error calling AfterAccept:", err) + log.Println("[Update] error calling AfterAcceptUpdate:", err) return } diff --git a/system/db/content.go b/system/db/content.go index d9096ae..81abd4f 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -36,6 +36,98 @@ func SetContent(target string, data url.Values) (int, error) { return update(ns, id, data) } +// UpdateContent inserts or updates values in the database. +// Updated content will be merged into existing values from the database. +// The `target` argument is a string made up of namespace:id (string:int) +func UpdateContent(target string, data url.Values) (int, error) { + t := strings.Split(target, ":") + ns, id := t[0], t[1] + + // check if content id == -1 (indicating new post). + // if so, run an insert which will assign the next auto incremented int. + // this is done because boltdb begins its bucket auto increment value at 0, + // which is the zero-value of an int in the Item struct field for ID. + // this is a problem when the original first post (with auto ID = 0) gets + // overwritten by any new post, originally having no ID, defauting to 0. + if id == "-1" { + return insert(ns, data) + } + + // retrieve existing content from the database + existingContent, err := Content(target) + if err != nil { + return 0, err + } + + // Unmarsal the existing values + s := item.Types[ns]() + + err = json.Unmarshal(existingContent, &s) + if err != nil { + log.Println("Error decoding json while sorting", ns, ":", err) + return 0, err + } + + var specifier string // i.e. __pending, __sorted, etc. + if strings.Contains(ns, "__") { + spec := strings.Split(ns, "__") + ns = spec[0] + specifier = "__" + spec[1] + } + + cid, err := strconv.Atoi(id) + if err != nil { + return 0, err + } + + // Don't allow the Item fields to be updated from form values + data.Del("id") + data.Del("uuid") + data.Del("slug") + + dec := schema.NewDecoder() + dec.SetAliasTag("json") // allows simpler struct tagging when creating a content type + dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct + err = dec.Decode(s, data) + if err != nil { + return 0, err + } + + j, err := json.Marshal(s) + if err != nil { + return 0, err + } + + err = store.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier)) + if err != nil { + return err + } + + err = b.Put([]byte(fmt.Sprintf("%d", cid)), j) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return 0, nil + } + + if specifier == "" { + go SortContent(ns) + } + + // update changes data, so invalidate client caching + err = InvalidateCache() + if err != nil { + return 0, err + } + + return cid, nil +} + func update(ns, id string, data url.Values) (int, error) { var specifier string // i.e. __pending, __sorted, etc. if strings.Contains(ns, "__") { diff --git a/system/item/item.go b/system/item/item.go index 5b41693..286842b 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -42,6 +42,9 @@ type Sortable interface { // to the different lifecycles/events a struct may encounter. Item implements // Hookable with no-ops so our user can override only whichever ones necessary. type Hookable interface { + BeforeAcceptUpdate(http.ResponseWriter, *http.Request) error + AfterAcceptUpdate(http.ResponseWriter, *http.Request) error + BeforeAccept(http.ResponseWriter, *http.Request) error AfterAccept(http.ResponseWriter, *http.Request) error @@ -132,6 +135,16 @@ func (i Item) String() string { return fmt.Sprintf("Item ID: %s", i.UniqueID()) } +// BeforeAcceptUpdate is a no-op to ensure structs which embed Item implement Hookable +func (i Item) BeforeAcceptUpdate(res http.ResponseWriter, req *http.Request) error { + return nil +} + +// AfterAcceptUpdate is a no-op to ensure structs which embed Item implement Hookable +func (i Item) AfterAcceptUpdate(res http.ResponseWriter, req *http.Request) error { + return nil +} + // BeforeAccept is a no-op to ensure structs which embed Item implement Hookable func (i Item) BeforeAccept(res http.ResponseWriter, req *http.Request) error { return nil -- cgit v1.2.3 From 266b7834c586f3c6be20d6772b36347d1f2a6b35 Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Thu, 2 Mar 2017 13:10:00 -0600 Subject: updated log message --- system/db/content.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index 81abd4f..8064b03 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -64,7 +64,7 @@ func UpdateContent(target string, data url.Values) (int, error) { err = json.Unmarshal(existingContent, &s) if err != nil { - log.Println("Error decoding json while sorting", ns, ":", err) + log.Println("Error decoding json while updating", ns, ":", err) return 0, err } -- cgit v1.2.3 From 92a96fe470b39ae5c16f66fce8023f04ad3a37ae Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Sun, 5 Mar 2017 22:11:42 -0600 Subject: consolidate UpdateContent into SetContent --- system/api/update.go | 6 +-- system/db/content.go | 125 +++++++++++++++++++-------------------------------- 2 files changed, 50 insertions(+), 81 deletions(-) (limited to 'system') diff --git a/system/api/update.go b/system/api/update.go index 00933fb..507d3af 100644 --- a/system/api/update.go +++ b/system/api/update.go @@ -52,8 +52,8 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { } id := req.URL.Query().Get("id") - if id == "" { - log.Println("[Update] attempt to submit update with missing id from:", req.RemoteAddr) + if !db.IsValidID(id) { + log.Println("[Update] attempt to submit update with missing or invalid id from:", req.RemoteAddr) res.WriteHeader(http.StatusBadRequest) return } @@ -160,7 +160,7 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { // set specifier for db bucket in case content is/isn't Trustable var spec string - _, err = db.UpdateContent(t+spec+":"+id, req.PostForm) + _, err = db.SetContent(t+spec+":"+id, req.PostForm) if err != nil { log.Println("[Update] error calling UpdateContent:", err) res.WriteHeader(http.StatusInternalServerError) diff --git a/system/db/content.go b/system/db/content.go index 8064b03..4517fbe 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -3,6 +3,7 @@ package db import ( "bytes" "encoding/json" + "errors" "fmt" "log" "net/url" @@ -17,29 +18,18 @@ import ( uuid "github.com/satori/go.uuid" ) -// SetContent inserts or updates values in the database. -// The `target` argument is a string made up of namespace:id (string:int) -func SetContent(target string, data url.Values) (int, error) { - t := strings.Split(target, ":") - ns, id := t[0], t[1] - - // check if content id == -1 (indicating new post). - // if so, run an insert which will assign the next auto incremented int. - // this is done because boltdb begins its bucket auto increment value at 0, - // which is the zero-value of an int in the Item struct field for ID. - // this is a problem when the original first post (with auto ID = 0) gets - // overwritten by any new post, originally having no ID, defauting to 0. - if id == "-1" { - return insert(ns, data) +func IsValidID(id string) bool { + // ID should be a non-negative integer. + // ID of -1 is special for new posts, not updates. + if i, err := strconv.Atoi(id); err != nil || i < 0 { + return false } - - return update(ns, id, data) + return true } -// UpdateContent inserts or updates values in the database. -// Updated content will be merged into existing values from the database. +// SetContent inserts or updates values in the database. // The `target` argument is a string made up of namespace:id (string:int) -func UpdateContent(target string, data url.Values) (int, error) { +func SetContent(target string, data url.Values) (int, error) { t := strings.Split(target, ":") ns, id := t[0], t[1] @@ -58,16 +48,11 @@ func UpdateContent(target string, data url.Values) (int, error) { if err != nil { return 0, err } + return update(ns, id, data, &existingContent) +} - // Unmarsal the existing values - s := item.Types[ns]() - - err = json.Unmarshal(existingContent, &s) - if err != nil { - log.Println("Error decoding json while updating", ns, ":", err) - return 0, err - } - +// update can support merge or replace behavior +func update(ns, id string, data url.Values, existingContent *[]byte) (int, error) { var specifier string // i.e. __pending, __sorted, etc. if strings.Contains(ns, "__") { spec := strings.Split(ns, "__") @@ -80,22 +65,17 @@ func UpdateContent(target string, data url.Values) (int, error) { return 0, err } - // Don't allow the Item fields to be updated from form values - data.Del("id") - data.Del("uuid") - data.Del("slug") - - dec := schema.NewDecoder() - dec.SetAliasTag("json") // allows simpler struct tagging when creating a content type - dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct - err = dec.Decode(s, data) - if err != nil { - return 0, err - } - - j, err := json.Marshal(s) - if err != nil { - return 0, err + var j []byte + if existingContent == nil { + j, err = postToJSON(ns, data) + if err != nil { + return 0, err + } + } else { + j, err = mergeData(ns, data, *existingContent) + if err != nil { + return 0, err + } } err = store.Update(func(tx *bolt.Tx) error { @@ -128,52 +108,41 @@ func UpdateContent(target string, data url.Values) (int, error) { return cid, nil } -func update(ns, id string, data url.Values) (int, error) { - var specifier string // i.e. __pending, __sorted, etc. - if strings.Contains(ns, "__") { - spec := strings.Split(ns, "__") - ns = spec[0] - specifier = "__" + spec[1] - } - - cid, err := strconv.Atoi(id) - if err != nil { - return 0, err +func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, error) { + var j []byte + t, ok := item.Types[ns] + if !ok { + return j, errors.New("Invalid type.") + // handle } - j, err := postToJSON(ns, data) + // Unmarsal the existing values + s := t() + err := json.Unmarshal(existingContent, &s) if err != nil { - return 0, err + log.Println("Error decoding json while updating", ns, ":", err) + return j, err } - err = store.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier)) - if err != nil { - return err - } - - err = b.Put([]byte(fmt.Sprintf("%d", cid)), j) - if err != nil { - return err - } + // Don't allow the Item fields to be updated from form values + data.Del("id") + data.Del("uuid") + data.Del("slug") - return nil - }) + dec := schema.NewDecoder() + dec.SetAliasTag("json") // allows simpler struct tagging when creating a content type + dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct + err = dec.Decode(s, data) if err != nil { - return 0, nil - } - - if specifier == "" { - go SortContent(ns) + return j, err } - // update changes data, so invalidate client caching - err = InvalidateCache() + j, err = json.Marshal(s) if err != nil { - return 0, err + return j, err } - return cid, nil + return j, nil } func insert(ns string, data url.Values) (int, error) { -- cgit v1.2.3 From 986a2130a59b25fdc7730b3a974d5f5338bd7cb1 Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Sun, 5 Mar 2017 22:35:41 -0600 Subject: updated logging --- system/api/update.go | 2 +- system/db/content.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'system') diff --git a/system/api/update.go b/system/api/update.go index 507d3af..da79330 100644 --- a/system/api/update.go +++ b/system/api/update.go @@ -162,7 +162,7 @@ func updateContentHandler(res http.ResponseWriter, req *http.Request) { _, err = db.SetContent(t+spec+":"+id, req.PostForm) if err != nil { - log.Println("[Update] error calling UpdateContent:", err) + log.Println("[Update] error calling SetContent:", err) res.WriteHeader(http.StatusInternalServerError) return } diff --git a/system/db/content.go b/system/db/content.go index 4517fbe..dd93c60 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -112,8 +112,8 @@ func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, erro var j []byte t, ok := item.Types[ns] if !ok { + log.Println("Type not found from namespace:", ns) return j, errors.New("Invalid type.") - // handle } // Unmarsal the existing values -- cgit v1.2.3 From ef28c19b8237e03bd521118b04109e9922c9d51d Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 7 Mar 2017 00:25:13 -0800 Subject: adding catch for ErrRecursivePush log and ignoring null values --- system/api/push.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'system') diff --git a/system/api/push.go b/system/api/push.go index b7c5642..a6402bc 100644 --- a/system/api/push.go +++ b/system/api/push.go @@ -7,6 +7,7 @@ import ( "github.com/ponzu-cms/ponzu/system/item" "github.com/tidwall/gjson" + "golang.org/x/net/http2" ) func push(res http.ResponseWriter, req *http.Request, pt func() interface{}, data []byte) { @@ -23,8 +24,12 @@ func push(res http.ResponseWriter, req *http.Request, pt func() interface{}, dat for i := range values { val := values[i] val.ForEach(func(k, v gjson.Result) bool { + if v.String() == "null" { + return true + } + err := pusher.Push(v.String(), nil) - if err != nil { + if err != nil && err != http2.ErrRecursivePush { log.Println("Error during Push of value:", v.String()) } -- cgit v1.2.3