From ade7c2d69c9884c493fbc2f66f6a7405c0f4f96b Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 13 Apr 2017 13:29:18 -0700 Subject: moving search into own package and renaming funcs throughout --- system/api/search.go | 5 +- system/db/content.go | 13 ++-- system/db/init.go | 3 +- system/db/search.go | 162 ------------------------------------------------ system/item/item.go | 4 +- system/search/search.go | 162 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 176 insertions(+), 173 deletions(-) delete mode 100644 system/db/search.go create mode 100644 system/search/search.go (limited to 'system') diff --git a/system/api/search.go b/system/api/search.go index c743d9f..9c7f0ae 100644 --- a/system/api/search.go +++ b/system/api/search.go @@ -8,6 +8,7 @@ import ( "github.com/ponzu-cms/ponzu/system/db" "github.com/ponzu-cms/ponzu/system/item" + "github.com/ponzu-cms/ponzu/system/search" ) func searchContentHandler(res http.ResponseWriter, req *http.Request) { @@ -42,8 +43,8 @@ func searchContentHandler(res http.ResponseWriter, req *http.Request) { } // execute search for query provided, if no index for type send 404 - matches, err := db.SearchType(t, q) - if err == db.ErrNoSearchIndex { + matches, err := search.TypeQuery(t, q) + if err == search.ErrNoIndex { res.WriteHeader(http.StatusNotFound) return } diff --git a/system/db/content.go b/system/db/content.go index fbdd8dd..b3e2642 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ponzu-cms/ponzu/system/item" + "github.com/ponzu-cms/ponzu/system/search" "github.com/boltdb/bolt" "github.com/gorilla/schema" @@ -125,9 +126,9 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error go func() { // update data in search index target := fmt.Sprintf("%s:%s", ns, id) - err = UpdateSearchIndex(target, j) + err = search.UpdateIndex(target, j) if err != nil { - log.Println("[search] UpdateSearchIndex Error:", err) + log.Println("[search] UpdateIndex Error:", err) } }() @@ -253,9 +254,9 @@ func insert(ns string, data url.Values) (int, error) { go func() { // add data to seach index target := fmt.Sprintf("%s:%s", ns, cid) - err = UpdateSearchIndex(target, j) + err = search.UpdateIndex(target, j) if err != nil { - log.Println("[search] UpdateSearchIndex Error:", err) + log.Println("[search] UpdateIndex Error:", err) } }() @@ -322,9 +323,9 @@ func DeleteContent(target string) error { // delete indexed data from search index if !strings.Contains(ns, "__") { target = fmt.Sprintf("%s:%s", ns, id) - err = DeleteSearchIndex(target) + err = search.DeleteIndex(target) if err != nil { - log.Println("[search] DeleteSearchIndex Error:", err) + log.Println("[search] DeleteIndex Error:", err) } } }() diff --git a/system/db/init.go b/system/db/init.go index 4e9c3cf..3fc35f5 100644 --- a/system/db/init.go +++ b/system/db/init.go @@ -4,6 +4,7 @@ import ( "log" "github.com/ponzu-cms/ponzu/system/item" + "github.com/ponzu-cms/ponzu/system/search" "github.com/boltdb/bolt" "github.com/nilslice/jwt" @@ -80,7 +81,7 @@ func Init() { go func() { for t := range item.Types { - err := MapSearchIndex(t) + err := search.MapIndex(t) if err != nil { log.Fatalln(err) return diff --git a/system/db/search.go b/system/db/search.go deleted file mode 100644 index b3d5fa1..0000000 --- a/system/db/search.go +++ /dev/null @@ -1,162 +0,0 @@ -package db - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/ponzu-cms/ponzu/system/item" - - "encoding/json" - - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/mapping" -) - -var ( - // Search tracks all search indices to use throughout system - Search map[string]bleve.Index - - // ErrNoSearchIndex is for failed checks for an index in Search map - ErrNoSearchIndex = errors.New("No search index found for type provided") -) - -// Searchable ... -type Searchable interface { - SearchMapping() (*mapping.IndexMappingImpl, error) - IndexContent() bool -} - -func init() { - Search = make(map[string]bleve.Index) -} - -// MapSearchIndex creates the mapping for a type and tracks the index to be used within -// the system for adding/deleting/checking data -func MapSearchIndex(typeName string) error { - // type assert for Searchable, get configuration (which can be overridden) - // by Ponzu user if defines own SearchMapping() - it, ok := item.Types[typeName] - if !ok { - return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) - } - s, ok := it().(Searchable) - if !ok { - return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName) - } - - // skip setting or using index for types that shouldn't be indexed - if !s.IndexContent() { - return nil - } - - mapping, err := s.SearchMapping() - if err != nil { - return err - } - - idxName := typeName + ".index" - var idx bleve.Index - - // check if index exists, use it or create new one - pwd, err := os.Getwd() - if err != nil { - return err - } - - searchPath := filepath.Join(pwd, "search") - - err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) - if err != nil { - return err - } - - idxPath := filepath.Join(searchPath, idxName) - if _, err = os.Stat(idxPath); os.IsNotExist(err) { - idx, err = bleve.New(idxPath, mapping) - if err != nil { - return err - } - idx.SetName(idxName) - } else { - idx, err = bleve.Open(idxPath) - if err != nil { - return err - } - } - - // add the type name to the index and track the index - Search[typeName] = idx - - return nil -} - -// UpdateSearchIndex sets data into a content type's search index at the given -// identifier -func UpdateSearchIndex(id string, data interface{}) error { - // check if there is a search index to work with - target := strings.Split(id, ":") - ns := target[0] - - idx, ok := Search[ns] - if ok { - // unmarshal json to struct, error if not registered - it, ok := item.Types[ns] - if !ok { - return fmt.Errorf("[search] UpdateSearchIndex Error: type '%s' doesn't exist", ns) - } - - p := it() - err := json.Unmarshal(data.([]byte), &p) - if err != nil { - return err - } - - // add data to search index - return idx.Index(id, p) - } - - return nil -} - -// DeleteSearchIndex removes data from a content type's search index at the -// given identifier -func DeleteSearchIndex(id string) error { - // check if there is a search index to work with - target := strings.Split(id, ":") - ns := target[0] - - idx, ok := Search[ns] - if ok { - // add data to search index - return idx.Delete(id) - } - - return nil -} - -// SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs, -// and an error. If there is no search index for the typeName (Type) provided, -// db.ErrNoSearchIndex will be returned as the error -func SearchType(typeName, query string) ([]string, error) { - idx, ok := Search[typeName] - if !ok { - return nil, ErrNoSearchIndex - } - - q := bleve.NewQueryStringQuery(query) - req := bleve.NewSearchRequest(q) - res, err := idx.Search(req) - if err != nil { - return nil, err - } - - var results []string - for _, hit := range res.Hits { - results = append(results, hit.ID) - } - - return results, nil -} diff --git a/system/item/item.go b/system/item/item.go index 8f392ca..227cd26 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -211,7 +211,7 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error { } // SearchMapping returns a default implementation of a Bleve IndexMappingImpl -// partially implements db.Searchable +// partially implements search.Searchable func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) { mapping := bleve.NewIndexMapping() mapping.StoreDynamic = false @@ -220,7 +220,7 @@ func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) { } // IndexContent determines if a type should be indexed for searching -// partially implements db.Searchable +// partially implements search.Searchable func (i Item) IndexContent() bool { return false } diff --git a/system/search/search.go b/system/search/search.go new file mode 100644 index 0000000..6d538ed --- /dev/null +++ b/system/search/search.go @@ -0,0 +1,162 @@ +package search + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ponzu-cms/ponzu/system/item" + + "encoding/json" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" +) + +var ( + // Search tracks all search indices to use throughout system + Search map[string]bleve.Index + + // ErrNoIndex is for failed checks for an index in Search map + ErrNoIndex = errors.New("No search index found for type provided") +) + +// Searchable ... +type Searchable interface { + SearchMapping() (*mapping.IndexMappingImpl, error) + IndexContent() bool +} + +func init() { + Search = make(map[string]bleve.Index) +} + +// MapIndex creates the mapping for a type and tracks the index to be used within +// the system for adding/deleting/checking data +func MapIndex(typeName string) error { + // type assert for Searchable, get configuration (which can be overridden) + // by Ponzu user if defines own SearchMapping() + it, ok := item.Types[typeName] + if !ok { + return fmt.Errorf("[search] MapIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) + } + s, ok := it().(Searchable) + if !ok { + return fmt.Errorf("[search] MapIndex Error: Item type %s doesn't implement search.Searchable", typeName) + } + + // skip setting or using index for types that shouldn't be indexed + if !s.IndexContent() { + return nil + } + + mapping, err := s.SearchMapping() + if err != nil { + return err + } + + idxName := typeName + ".index" + var idx bleve.Index + + // check if index exists, use it or create new one + pwd, err := os.Getwd() + if err != nil { + return err + } + + searchPath := filepath.Join(pwd, "search") + + err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) + if err != nil { + return err + } + + idxPath := filepath.Join(searchPath, idxName) + if _, err = os.Stat(idxPath); os.IsNotExist(err) { + idx, err = bleve.New(idxPath, mapping) + if err != nil { + return err + } + idx.SetName(idxName) + } else { + idx, err = bleve.Open(idxPath) + if err != nil { + return err + } + } + + // add the type name to the index and track the index + Search[typeName] = idx + + return nil +} + +// UpdateIndex sets data into a content type's search index at the given +// identifier +func UpdateIndex(id string, data interface{}) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // unmarshal json to struct, error if not registered + it, ok := item.Types[ns] + if !ok { + return fmt.Errorf("[search] UpdateIndex Error: type '%s' doesn't exist", ns) + } + + p := it() + err := json.Unmarshal(data.([]byte), &p) + if err != nil { + return err + } + + // add data to search index + return idx.Index(id, p) + } + + return nil +} + +// DeleteIndex removes data from a content type's search index at the +// given identifier +func DeleteIndex(id string) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // add data to search index + return idx.Delete(id) + } + + return nil +} + +// TypeQuery conducts a search and returns a set of Ponzu "targets", Type:ID pairs, +// and an error. If there is no search index for the typeName (Type) provided, +// db.ErrNoIndex will be returned as the error +func TypeQuery(typeName, query string) ([]string, error) { + idx, ok := Search[typeName] + if !ok { + return nil, ErrNoIndex + } + + q := bleve.NewQueryStringQuery(query) + req := bleve.NewSearchRequest(q) + res, err := idx.Search(req) + if err != nil { + return nil, err + } + + var results []string + for _, hit := range res.Hits { + results = append(results, hit.ID) + } + + return results, nil +} -- cgit v1.2.3 From 6097361fc47a361800ad35a11d6fe3719b2ae0a4 Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Fri, 14 Apr 2017 15:29:21 -0500 Subject: improved throttle to ensure content is sorted --- system/db/content.go | 59 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 25 deletions(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index b3e2642..e573e79 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/ponzu-cms/ponzu/system/item" @@ -567,41 +568,49 @@ func Query(namespace string, opts QueryOptions) (int, [][]byte) { var sortContentCalls = make(map[string]time.Time) var waitDuration = time.Millisecond * 2000 +var sortMutex = &sync.Mutex{} -func enoughTime(key string, withDelay bool) bool { +func setLastInvocation(key string) { + sortMutex.Lock() + sortContentCalls[key] = time.Now() + sortMutex.Unlock() +} + +func lastInvocation(key string) (time.Time, bool) { + sortMutex.Lock() last, ok := sortContentCalls[key] + sortMutex.Unlock() + return last, ok +} + +func enoughTime(key string) bool { + last, ok := lastInvocation(key) if !ok { // no invocation yet // track next invocation - sortContentCalls[key] = time.Now() + setLastInvocation(key) return true } - // if our required wait time has not been met, return false - if !time.Now().After(last.Add(waitDuration)) { - return false + // if our required wait time has been met, return true + if time.Now().After(last.Add(waitDuration)) { + setLastInvocation(key) + return true } - // dispatch a delayed envocation in case no additional one follows - if withDelay { - go func() { - select { - case <-time.After(waitDuration): - if enoughTime(key, false) { - // track next invocation - sortContentCalls[key] = time.Now() - SortContent(key) - } else { - // retrigger - SortContent(key) - } + // dispatch a delayed invocation in case no additional one follows + go func() { + lastInvocationBeforeTicker, _ := lastInvocation(key) // zero value can be handled, no need for ok + enoughTicker := time.NewTicker(waitDuration) + select { + case <-enoughTicker.C: + lastInvocationAfterTicker, _ := lastInvocation(key) + if !lastInvocationAfterTicker.After(lastInvocationBeforeTicker) { + SortContent(key) } - }() - } - - // track next invocation - sortContentCalls[key] = time.Now() - return true + } + }() + return false } // SortContent sorts all content of the type supplied as the namespace by time, @@ -609,7 +618,7 @@ func enoughTime(key string, withDelay bool) bool { // Should be called from a goroutine after SetContent is successful func SortContent(namespace string) { // wait if running too frequently per namespace - if !enoughTime(namespace, true) { + if !enoughTime(namespace) { return } -- cgit v1.2.3 From 19c8b41621a73d19ba74f3b2fc82ae53206d242c Mon Sep 17 00:00:00 2001 From: Kevin Keuning Date: Fri, 14 Apr 2017 16:18:12 -0500 Subject: simpler throttle --- system/db/content.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index e573e79..b503a60 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -600,16 +600,15 @@ func enoughTime(key string) bool { // dispatch a delayed invocation in case no additional one follows go func() { - lastInvocationBeforeTicker, _ := lastInvocation(key) // zero value can be handled, no need for ok - enoughTicker := time.NewTicker(waitDuration) - select { - case <-enoughTicker.C: - lastInvocationAfterTicker, _ := lastInvocation(key) - if !lastInvocationAfterTicker.After(lastInvocationBeforeTicker) { - SortContent(key) - } + lastInvocationBeforeTimer, _ := lastInvocation(key) // zero value can be handled, no need for ok + enoughTimer := time.NewTimer(waitDuration) + <-enoughTimer.C + lastInvocationAfterTimer, _ := lastInvocation(key) + if !lastInvocationAfterTimer.After(lastInvocationBeforeTimer) { + SortContent(key) } }() + return false } -- cgit v1.2.3