From a7bdae0fb109f6fbe42afda6800338b76244a82c Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 10:49:49 -0700 Subject: add mapping for search indices per content type --- system/db/init.go | 6 ++++++ system/db/search.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 system/db/search.go (limited to 'system') diff --git a/system/db/init.go b/system/db/init.go index 9125d3b..e59e7c4 100644 --- a/system/db/init.go +++ b/system/db/init.go @@ -80,6 +80,12 @@ func Init() { go func() { for t := range item.Types { + err := MapIndex(t) + if err != nil { + log.Fatalln("[search] Error:", err) + return + } + SortContent(t) } }() diff --git a/system/db/search.go b/system/db/search.go new file mode 100644 index 0000000..d5fc9f5 --- /dev/null +++ b/system/db/search.go @@ -0,0 +1,46 @@ +package db + +import ( + "os" + "path/filepath" + + "github.com/blevesearch/bleve" +) + +// Search tracks all search indices to use throughout system +var Search map[string]bleve.Index + +func init() { + Search = make(map[string]bleve.Index) +} + +// MapIndex creates the mapping for a type and tracks the index to be used within +// the system for adding/deleting/checking data +func MapIndex(typeName string) error { + mapping := bleve.NewIndexMapping() + mapping.StoreDynamic = false + idxFile := typeName + ".index" + var idx bleve.Index + + // check if index exists, use it or create new one + pwd, err := os.Getwd() + if err != nil { + return err + } + if _, err = os.Stat(filepath.Join(pwd, idxFile)); os.IsNotExist(err) { + idx, err = bleve.New(idxFile, mapping) + if err != nil { + return err + } + } else { + idx, err = bleve.Open(idxFile) + if err != nil { + return err + } + } + + // add the type name to the index and track the index + Search[typeName] = idx + + return nil +} -- cgit v1.2.3 From 38a6c611da7d5f4aa3a1d4fbb94ab5371b9cae5f Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 12:27:12 -0700 Subject: adding json values to search index on insert and update --- system/db/content.go | 20 ++++++++++++++++++-- system/db/search.go | 25 +++++++++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index 97ea9b4..cc3382f 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -121,6 +121,13 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error return 0, err } + // add data to search index + target := fmt.Sprintf("%s:%s", ns, id) + err = Search[ns].Index(target, j) + if err != nil { + return 0, err + } + return cid, nil } @@ -169,6 +176,8 @@ func insert(ns string, data url.Values) (int, error) { specifier = "__" + spec[1] } + var j []byte + var cid string err := store.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier)) if err != nil { @@ -181,7 +190,7 @@ func insert(ns string, data url.Values) (int, error) { if err != nil { return err } - cid := strconv.FormatUint(id, 10) + cid = strconv.FormatUint(id, 10) effectedID, err = strconv.Atoi(cid) if err != nil { return err @@ -197,7 +206,7 @@ func insert(ns string, data url.Values) (int, error) { data.Set("__specifier", specifier) } - j, err := postToJSON(ns, data) + j, err = postToJSON(ns, data) if err != nil { return err } @@ -238,6 +247,13 @@ func insert(ns string, data url.Values) (int, error) { return 0, err } + // add data to search index + target := fmt.Sprintf("%s:%s", ns, cid) + err = Search[ns].Index(target, j) + if err != nil { + return 0, err + } + return effectedID, nil } diff --git a/system/db/search.go b/system/db/search.go index d5fc9f5..8b642ff 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -5,11 +5,17 @@ import ( "path/filepath" "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" ) // Search tracks all search indices to use throughout system var Search map[string]bleve.Index +// Searchable ... +type Searchable interface { + SearchMapping() *mapping.IndexMappingImpl +} + func init() { Search = make(map[string]bleve.Index) } @@ -17,9 +23,12 @@ func init() { // MapIndex creates the mapping for a type and tracks the index to be used within // the system for adding/deleting/checking data func MapIndex(typeName string) error { + // TODO: type assert for Searchable, get configuration (which can be overridden) + // by Ponzu user if defines own SearchMapping() + mapping := bleve.NewIndexMapping() mapping.StoreDynamic = false - idxFile := typeName + ".index" + idxPath := typeName + ".index" var idx bleve.Index // check if index exists, use it or create new one @@ -27,13 +36,21 @@ func MapIndex(typeName string) error { if err != nil { return err } - if _, err = os.Stat(filepath.Join(pwd, idxFile)); os.IsNotExist(err) { - idx, err = bleve.New(idxFile, mapping) + + searchPath := filepath.Join(pwd, "search") + + err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) + if err != nil { + return err + } + + if _, err = os.Stat(filepath.Join(searchPath, idxPath)); os.IsNotExist(err) { + idx, err = bleve.New(idxPath, mapping) if err != nil { return err } } else { - idx, err = bleve.Open(idxFile) + idx, err = bleve.Open(idxPath) if err != nil { return err } -- cgit v1.2.3 From 582c47da170b79e6f4e64fe2ed85a3931e519c2b Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 12:31:04 -0700 Subject: update index path to include the search dir --- system/db/search.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'system') diff --git a/system/db/search.go b/system/db/search.go index 8b642ff..2474e19 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -28,7 +28,7 @@ func MapIndex(typeName string) error { mapping := bleve.NewIndexMapping() mapping.StoreDynamic = false - idxPath := typeName + ".index" + idxName := typeName + ".index" var idx bleve.Index // check if index exists, use it or create new one @@ -44,7 +44,8 @@ func MapIndex(typeName string) error { return err } - if _, err = os.Stat(filepath.Join(searchPath, idxPath)); os.IsNotExist(err) { + idxPath := filepath.Join(searchPath, idxName) + if _, err = os.Stat(idxPath); os.IsNotExist(err) { idx, err = bleve.New(idxPath, mapping) if err != nil { return err -- cgit v1.2.3 From f7a1c1afd6a0bf5708530c31da55035322f37ff4 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 12:38:44 -0700 Subject: indexing string data instead of json []byte --- system/db/content.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index cc3382f..7667831 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -123,7 +123,7 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error // add data to search index target := fmt.Sprintf("%s:%s", ns, id) - err = Search[ns].Index(target, j) + err = Search[ns].Index(target, string(j)) if err != nil { return 0, err } @@ -249,7 +249,7 @@ func insert(ns string, data url.Values) (int, error) { // add data to search index target := fmt.Sprintf("%s:%s", ns, cid) - err = Search[ns].Index(target, j) + err = Search[ns].Index(target, string(j)) if err != nil { return 0, err } -- cgit v1.2.3 From 338d354222a60fdd4c5e9c9493932f684bae6827 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 12:55:18 -0700 Subject: adding default implementation of db.Searchable --- system/db/search.go | 16 +++++++++++++--- system/item/item.go | 11 +++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'system') diff --git a/system/db/search.go b/system/db/search.go index 2474e19..743a904 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -1,11 +1,13 @@ package db import ( + "fmt" "os" "path/filepath" "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" + "github.com/ponzu-cms/ponzu/system/item" ) // Search tracks all search indices to use throughout system @@ -23,11 +25,19 @@ func init() { // MapIndex creates the mapping for a type and tracks the index to be used within // the system for adding/deleting/checking data func MapIndex(typeName string) error { - // TODO: type assert for Searchable, get configuration (which can be overridden) + // type assert for Searchable, get configuration (which can be overridden) // by Ponzu user if defines own SearchMapping() + it, ok := item.Types[typeName] + if !ok { + return fmt.Errorf("Failed to MapIndex for %s, type doesn't exist", typeName) + } + s, ok := it().(Searchable) + if !ok { + return fmt.Errorf("Item type %s doesn't implement db.Searchable", typeName) + } + + mapping := s.SearchMapping() - mapping := bleve.NewIndexMapping() - mapping.StoreDynamic = false idxName := typeName + ".index" var idx bleve.Index diff --git a/system/item/item.go b/system/item/item.go index 99d70a8..24ac003 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -7,6 +7,8 @@ import ( "strings" "unicode" + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" uuid "github.com/satori/go.uuid" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" @@ -208,6 +210,15 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error { return nil } +// SearchMapping returns a default implementation of a Bleve IndexMappingImpl +// partially implements db.Searchable +func SearchMapping() *mapping.IndexMappingImpl { + mapping := bleve.NewIndexMapping() + mapping.StoreDynamic = false + + return mapping +} + // Slug returns a URL friendly string from the title of a post item func Slug(i Identifiable) (string, error) { // get the name of the post item -- cgit v1.2.3 From bf15dcfa48a46991f3c6d90f50aa491b5a15d5df Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 12:57:54 -0700 Subject: default implementation fix --- system/item/item.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system') diff --git a/system/item/item.go b/system/item/item.go index 24ac003..5babf35 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -212,7 +212,7 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error { // SearchMapping returns a default implementation of a Bleve IndexMappingImpl // partially implements db.Searchable -func SearchMapping() *mapping.IndexMappingImpl { +func (i Item) SearchMapping() *mapping.IndexMappingImpl { mapping := bleve.NewIndexMapping() mapping.StoreDynamic = false -- cgit v1.2.3 From ba201f640c883d5ed2f3e1f68daff3a7ee4c8b9f Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Thu, 6 Apr 2017 17:53:34 -0700 Subject: renaming and add Delete operation for search index --- system/db/content.go | 17 ++++++++++++----- system/db/init.go | 2 +- system/db/search.go | 47 +++++++++++++++++++++++++++++++++++++++++++---- system/item/item.go | 4 ++-- system/item/types.go | 3 +++ 5 files changed, 61 insertions(+), 12 deletions(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index 7667831..a29ef20 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -121,9 +121,8 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error return 0, err } - // add data to search index target := fmt.Sprintf("%s:%s", ns, id) - err = Search[ns].Index(target, string(j)) + err = UpdateSearchIndex(target, string(j)) if err != nil { return 0, err } @@ -135,7 +134,7 @@ func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, erro var j []byte t, ok := item.Types[ns] if !ok { - return nil, fmt.Errorf("namespace type not found:", ns) + return nil, fmt.Errorf("Namespace type not found: %s", ns) } // Unmarsal the existing values @@ -247,9 +246,8 @@ func insert(ns string, data url.Values) (int, error) { return 0, err } - // add data to search index target := fmt.Sprintf("%s:%s", ns, cid) - err = Search[ns].Index(target, string(j)) + err = UpdateSearchIndex(target, string(j)) if err != nil { return 0, err } @@ -313,6 +311,15 @@ func DeleteContent(target string) error { return err } + // delete indexed data from search index + if !strings.Contains(ns, "__") { + target = fmt.Sprintf("%s:%s", ns, id) + err = DeleteSearchIndex(target) + if err != nil { + return err + } + } + // exception to typical "run in goroutine" pattern: // we want to have an updated admin view as soon as this is deleted, so // in some cases, the delete and redirect is faster than the sort, diff --git a/system/db/init.go b/system/db/init.go index e59e7c4..a598fa7 100644 --- a/system/db/init.go +++ b/system/db/init.go @@ -80,7 +80,7 @@ func Init() { go func() { for t := range item.Types { - err := MapIndex(t) + err := MapSearchIndex(t) if err != nil { log.Fatalln("[search] Error:", err) return diff --git a/system/db/search.go b/system/db/search.go index 743a904..7b7a203 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" @@ -15,16 +16,16 @@ var Search map[string]bleve.Index // Searchable ... type Searchable interface { - SearchMapping() *mapping.IndexMappingImpl + SearchMapping() (*mapping.IndexMappingImpl, error) } func init() { Search = make(map[string]bleve.Index) } -// MapIndex creates the mapping for a type and tracks the index to be used within +// MapSearchIndex creates the mapping for a type and tracks the index to be used within // the system for adding/deleting/checking data -func MapIndex(typeName string) error { +func MapSearchIndex(typeName string) error { // type assert for Searchable, get configuration (which can be overridden) // by Ponzu user if defines own SearchMapping() it, ok := item.Types[typeName] @@ -36,7 +37,13 @@ func MapIndex(typeName string) error { return fmt.Errorf("Item type %s doesn't implement db.Searchable", typeName) } - mapping := s.SearchMapping() + mapping, err := s.SearchMapping() + if err == item.ErrNoSearchMapping { + return nil + } + if err != nil { + return err + } idxName := typeName + ".index" var idx bleve.Index @@ -72,3 +79,35 @@ func MapIndex(typeName string) error { return nil } + +// UpdateSearchIndex sets data into a content type's search index at the given +// identifier +func UpdateSearchIndex(id string, data interface{}) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // add data to search index + return idx.Index(id, data) + } + + return nil +} + +// DeleteSearchIndex removes data from a content type's search index at the +// given identifier +func DeleteSearchIndex(id string) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // add data to search index + return idx.Delete(id) + } + + return nil +} diff --git a/system/item/item.go b/system/item/item.go index 5babf35..4750ef5 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -212,11 +212,11 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error { // SearchMapping returns a default implementation of a Bleve IndexMappingImpl // partially implements db.Searchable -func (i Item) SearchMapping() *mapping.IndexMappingImpl { +func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) { mapping := bleve.NewIndexMapping() mapping.StoreDynamic = false - return mapping + return mapping, nil } // Slug returns a URL friendly string from the title of a post item diff --git a/system/item/types.go b/system/item/types.go index bcae58a..dbf13af 100644 --- a/system/item/types.go +++ b/system/item/types.go @@ -26,6 +26,9 @@ var ( // if requested by a valid admin or user ErrAllowHiddenItem = errors.New(`Allow hidden item`) + // ErrNoSearchMapping can be used to tell the system not to create an index mapping + ErrNoSearchMapping = errors.New(`No search mapping for item`) + // Types is a map used to reference a type name to its actual Editable type // mainly for lookups in /admin route based utilities Types map[string]func() interface{} -- cgit v1.2.3 From afe92b2bcd3b8e914496006a414ad9b95dba3f0b Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Fri, 7 Apr 2017 20:36:07 -0700 Subject: add initial implementation of api search handler with full-text search by type --- system/api/search.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ system/api/server.go | 2 ++ system/db/content.go | 62 +++++++++++++++++++++++++++------------ system/db/init.go | 2 +- system/db/search.go | 39 ++++++++++++++++++++++--- 5 files changed, 163 insertions(+), 24 deletions(-) create mode 100644 system/api/search.go (limited to 'system') diff --git a/system/api/search.go b/system/api/search.go new file mode 100644 index 0000000..ae6ac1c --- /dev/null +++ b/system/api/search.go @@ -0,0 +1,82 @@ +package api + +import ( + "encoding/json" + "log" + "net/http" + "net/url" + + "github.com/ponzu-cms/ponzu/system/db" + "github.com/ponzu-cms/ponzu/system/item" +) + +func searchContentHandler(res http.ResponseWriter, req *http.Request) { + qs := req.URL.Query() + t := qs.Get("type") + // type must be set, future version may compile multi-type result set + if t == "" { + res.WriteHeader(http.StatusBadRequest) + return + } + + it, ok := item.Types[t] + if !ok { + res.WriteHeader(http.StatusBadRequest) + return + } + + if hide(it(), res, req) { + return + } + + q, err := url.QueryUnescape(qs.Get("q")) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + // q must be set + if q == "" { + res.WriteHeader(http.StatusBadRequest) + return + } + + // execute search for query provided, if no index for type send 404 + matches, err := db.SearchType(t, q) + if err == db.ErrNoSearchIndex { + res.WriteHeader(http.StatusBadRequest) + return + } + if err != nil { + log.Println("[search] Error:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + // respond with json formatted results + bb, err := db.ContentMulti(matches) + if err != nil { + log.Println("[search] Error:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + var result = []json.RawMessage{} + for i := range bb { + result = append(result, bb[i]) + } + + j, err := fmtJSON(result...) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + j, err = omit(it(), j) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + sendData(res, req, j) +} diff --git a/system/api/server.go b/system/api/server.go index c568877..209ddaa 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -13,4 +13,6 @@ func Run() { http.HandleFunc("/api/content/update", Record(CORS(updateContentHandler))) http.HandleFunc("/api/content/delete", Record(CORS(deleteContentHandler))) + + http.HandleFunc("/api/search", Record(CORS(searchContentHandler))) } diff --git a/system/db/content.go b/system/db/content.go index a29ef20..a16c267 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -10,10 +10,9 @@ import ( "strconv" "strings" - "github.com/ponzu-cms/ponzu/system/item" - "github.com/boltdb/bolt" "github.com/gorilla/schema" + "github.com/ponzu-cms/ponzu/system/item" uuid "github.com/satori/go.uuid" ) @@ -121,11 +120,14 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error return 0, err } - target := fmt.Sprintf("%s:%s", ns, id) - err = UpdateSearchIndex(target, string(j)) - if err != nil { - return 0, err - } + go func() { + // update data in search index + target := fmt.Sprintf("%s:%s", ns, id) + err = UpdateSearchIndex(target, string(j)) + if err != nil { + log.Println("[search] UpdateSearchIndex Error:", err) + } + }() return cid, nil } @@ -246,11 +248,14 @@ func insert(ns string, data url.Values) (int, error) { return 0, err } - target := fmt.Sprintf("%s:%s", ns, cid) - err = UpdateSearchIndex(target, string(j)) - if err != nil { - return 0, err - } + go func() { + // add data to seach index + target := fmt.Sprintf("%s:%s", ns, cid) + err = UpdateSearchIndex(target, string(j)) + if err != nil { + log.Println("[search] UpdateSearchIndex Error:", err) + } + }() return effectedID, nil } @@ -311,14 +316,16 @@ func DeleteContent(target string) error { return err } - // delete indexed data from search index - if !strings.Contains(ns, "__") { - target = fmt.Sprintf("%s:%s", ns, id) - err = DeleteSearchIndex(target) - if err != nil { - return err + go func() { + // delete indexed data from search index + if !strings.Contains(ns, "__") { + target = fmt.Sprintf("%s:%s", ns, id) + err = DeleteSearchIndex(target) + if err != nil { + log.Println("[search] DeleteSearchIndex Error:", err) + } } - } + }() // exception to typical "run in goroutine" pattern: // we want to have an updated admin view as soon as this is deleted, so @@ -357,6 +364,23 @@ func Content(target string) ([]byte, error) { return val.Bytes(), nil } +// ContentMulti returns a set of content based on the the targets / identifiers +// provided in Ponzu target string format: Type:ID +// NOTE: All targets should be of the same type +func ContentMulti(targets []string) ([][]byte, error) { + var contents [][]byte + for i := range targets { + b, err := Content(targets[i]) + if err != nil { + return nil, err + } + + contents = append(contents, b) + } + + return contents, nil +} + // ContentBySlug does a lookup in the content index to find the type and id of // the requested content. Subsequently, issues the lookup in the type bucket and // returns the the type and data at that ID or nil if nothing exists. diff --git a/system/db/init.go b/system/db/init.go index a598fa7..4e9c3cf 100644 --- a/system/db/init.go +++ b/system/db/init.go @@ -82,7 +82,7 @@ func Init() { for t := range item.Types { err := MapSearchIndex(t) if err != nil { - log.Fatalln("[search] Error:", err) + log.Fatalln(err) return } diff --git a/system/db/search.go b/system/db/search.go index 7b7a203..26e4321 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -1,6 +1,7 @@ package db import ( + "errors" "fmt" "os" "path/filepath" @@ -11,8 +12,13 @@ import ( "github.com/ponzu-cms/ponzu/system/item" ) -// Search tracks all search indices to use throughout system -var Search map[string]bleve.Index +var ( + // Search tracks all search indices to use throughout system + Search map[string]bleve.Index + + // ErrNoSearchIndex is for failed checks for an index in Search map + ErrNoSearchIndex = errors.New("No search index found for type provided") +) // Searchable ... type Searchable interface { @@ -30,11 +36,11 @@ func MapSearchIndex(typeName string) error { // by Ponzu user if defines own SearchMapping() it, ok := item.Types[typeName] if !ok { - return fmt.Errorf("Failed to MapIndex for %s, type doesn't exist", typeName) + return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) } s, ok := it().(Searchable) if !ok { - return fmt.Errorf("Item type %s doesn't implement db.Searchable", typeName) + return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName) } mapping, err := s.SearchMapping() @@ -67,6 +73,7 @@ func MapSearchIndex(typeName string) error { if err != nil { return err } + idx.SetName(idxName) } else { idx, err = bleve.Open(idxPath) if err != nil { @@ -111,3 +118,27 @@ func DeleteSearchIndex(id string) error { return nil } + +// SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs, +// and an error. If there is no search index for the typeName (Type) provided, +// db.ErrNoSearchIndex will be returned as the error +func SearchType(typeName, query string) ([]string, error) { + idx, ok := Search[typeName] + if !ok { + return nil, ErrNoSearchIndex + } + + q := bleve.NewQueryStringQuery(query) + req := bleve.NewSearchRequest(q) + res, err := idx.Search(req) + if err != nil { + return nil, err + } + + var results []string + for _, hit := range res.Hits { + results = append(results, hit.ID) + } + + return results, nil +} -- cgit v1.2.3 From 7603efc51e8b688dcfb97173659bc385d6dd92ff Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Fri, 7 Apr 2017 20:37:56 -0700 Subject: import layout style fix --- system/db/content.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'system') diff --git a/system/db/content.go b/system/db/content.go index a16c267..49cba87 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -10,9 +10,10 @@ import ( "strconv" "strings" + "github.com/ponzu-cms/ponzu/system/item" + "github.com/boltdb/bolt" "github.com/gorilla/schema" - "github.com/ponzu-cms/ponzu/system/item" uuid "github.com/satori/go.uuid" ) -- cgit v1.2.3 From 3f1d6b5df128973eb7c4c8a92c2b34fde2fb1ba0 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Fri, 7 Apr 2017 20:39:53 -0700 Subject: import layout style fix --- system/db/search.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'system') diff --git a/system/db/search.go b/system/db/search.go index 26e4321..3e7a9d6 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -7,9 +7,10 @@ import ( "path/filepath" "strings" + "github.com/ponzu-cms/ponzu/system/item" + "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" - "github.com/ponzu-cms/ponzu/system/item" ) var ( -- cgit v1.2.3 From b961e74656b0db97d5b3c14295ca8c6ba16b4424 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 11 Apr 2017 14:51:22 -0700 Subject: change implementation of search to opt-in per type, less risky if sensative data is added --- system/api/search.go | 2 +- system/db/content.go | 4 ++-- system/db/search.go | 22 +++++++++++++++++++++- system/item/item.go | 6 ++++++ 4 files changed, 30 insertions(+), 4 deletions(-) (limited to 'system') diff --git a/system/api/search.go b/system/api/search.go index ae6ac1c..25a1bae 100644 --- a/system/api/search.go +++ b/system/api/search.go @@ -44,7 +44,7 @@ func searchContentHandler(res http.ResponseWriter, req *http.Request) { // execute search for query provided, if no index for type send 404 matches, err := db.SearchType(t, q) if err == db.ErrNoSearchIndex { - res.WriteHeader(http.StatusBadRequest) + res.WriteHeader(http.StatusNotFound) return } if err != nil { diff --git a/system/db/content.go b/system/db/content.go index 49cba87..5db2896 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -124,7 +124,7 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error go func() { // update data in search index target := fmt.Sprintf("%s:%s", ns, id) - err = UpdateSearchIndex(target, string(j)) + err = UpdateSearchIndex(target, j) if err != nil { log.Println("[search] UpdateSearchIndex Error:", err) } @@ -252,7 +252,7 @@ func insert(ns string, data url.Values) (int, error) { go func() { // add data to seach index target := fmt.Sprintf("%s:%s", ns, cid) - err = UpdateSearchIndex(target, string(j)) + err = UpdateSearchIndex(target, j) if err != nil { log.Println("[search] UpdateSearchIndex Error:", err) } diff --git a/system/db/search.go b/system/db/search.go index 3e7a9d6..84a3828 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -9,6 +9,8 @@ import ( "github.com/ponzu-cms/ponzu/system/item" + "encoding/json" + "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" ) @@ -24,6 +26,7 @@ var ( // Searchable ... type Searchable interface { SearchMapping() (*mapping.IndexMappingImpl, error) + IndexContent() bool } func init() { @@ -44,6 +47,11 @@ func MapSearchIndex(typeName string) error { return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName) } + // skip setting or using index for types that shouldn't be indexed + if !s.IndexContent() { + return nil + } + mapping, err := s.SearchMapping() if err == item.ErrNoSearchMapping { return nil @@ -97,8 +105,20 @@ func UpdateSearchIndex(id string, data interface{}) error { idx, ok := Search[ns] if ok { + // unmarshal json to struct, error if not registered + it, ok := item.Types[ns] + if !ok { + return fmt.Errorf("[search] UpdateSearchIndex Error: type '%s' doesn't exist", ns) + } + + p := it() + err := json.Unmarshal(data.([]byte), &p) + if err != nil { + return err + } + // add data to search index - return idx.Index(id, data) + return idx.Index(id, p) } return nil diff --git a/system/item/item.go b/system/item/item.go index 4750ef5..8f392ca 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -219,6 +219,12 @@ func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) { return mapping, nil } +// IndexContent determines if a type should be indexed for searching +// partially implements db.Searchable +func (i Item) IndexContent() bool { + return false +} + // Slug returns a URL friendly string from the title of a post item func Slug(i Identifiable) (string, error) { // get the name of the post item -- cgit v1.2.3 From 715b2e5b32a4603e1eb580133b9ae1a5837760fc Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 11 Apr 2017 18:31:54 -0700 Subject: adding push support to search, only first matched result --- system/api/search.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'system') diff --git a/system/api/search.go b/system/api/search.go index 25a1bae..c743d9f 100644 --- a/system/api/search.go +++ b/system/api/search.go @@ -61,6 +61,11 @@ func searchContentHandler(res http.ResponseWriter, req *http.Request) { return } + // if we have matches, push the first as its matched by relevance + if len(bb) > 0 { + push(res, req, it, bb[0]) + } + var result = []json.RawMessage{} for i := range bb { result = append(result, bb[i]) -- cgit v1.2.3 From fa7803984062f00feec897205a0a15cc79b696c6 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 11 Apr 2017 18:58:07 -0700 Subject: remoeve ErrNoSearchIndex since the implementation is disable by default --- system/db/search.go | 3 --- system/item/types.go | 3 --- 2 files changed, 6 deletions(-) (limited to 'system') diff --git a/system/db/search.go b/system/db/search.go index 84a3828..b3d5fa1 100644 --- a/system/db/search.go +++ b/system/db/search.go @@ -53,9 +53,6 @@ func MapSearchIndex(typeName string) error { } mapping, err := s.SearchMapping() - if err == item.ErrNoSearchMapping { - return nil - } if err != nil { return err } diff --git a/system/item/types.go b/system/item/types.go index dbf13af..bcae58a 100644 --- a/system/item/types.go +++ b/system/item/types.go @@ -26,9 +26,6 @@ var ( // if requested by a valid admin or user ErrAllowHiddenItem = errors.New(`Allow hidden item`) - // ErrNoSearchMapping can be used to tell the system not to create an index mapping - ErrNoSearchMapping = errors.New(`No search mapping for item`) - // Types is a map used to reference a type name to its actual Editable type // mainly for lookups in /admin route based utilities Types map[string]func() interface{} -- cgit v1.2.3