diff options
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | system/api/search.go | 87 | ||||
| -rw-r--r-- | system/api/server.go | 2 | ||||
| -rw-r--r-- | system/db/content.go | 54 | ||||
| -rw-r--r-- | system/db/init.go | 6 | ||||
| -rw-r--r-- | system/db/search.go | 162 | ||||
| -rw-r--r-- | system/item/item.go | 17 |
7 files changed, 326 insertions, 3 deletions
@@ -295,6 +295,7 @@ $ ponzu --dev -fork=github.com/nilslice/ponzu new /path/to/new/project - [golang.org/x/text/transform](https://golang.org/x/text/transform) - [golang.org/x/crypto/bcrypt](https://golang.org/x/crypto/bcrypt) - [golang.org/x/net/http2](https://golang.org/x/net/http2) +- [github.com/blevesearch/bleve](https://github.com/blevesearch/bleve) - [github.com/nilslice/jwt](https://github.com/nilslice/jwt) - [github.com/nilslice/email](https://github.com/nilslice/email) - [github.com/gorilla/schema](https://github.com/gorilla/schema) diff --git a/system/api/search.go b/system/api/search.go new file mode 100644 index 0000000..c743d9f --- /dev/null +++ b/system/api/search.go @@ -0,0 +1,87 @@ +package api + +import ( + "encoding/json" + "log" + "net/http" + "net/url" + + "github.com/ponzu-cms/ponzu/system/db" + "github.com/ponzu-cms/ponzu/system/item" +) + +func searchContentHandler(res http.ResponseWriter, req *http.Request) { + qs := req.URL.Query() + t := qs.Get("type") + // type must be set, future version may compile multi-type result set + if t == "" { + res.WriteHeader(http.StatusBadRequest) + return + } + + it, ok := item.Types[t] + if !ok { + res.WriteHeader(http.StatusBadRequest) + return + } + + if hide(it(), res, req) { + return + } + + q, err := url.QueryUnescape(qs.Get("q")) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + // q must be set + if q == "" { + res.WriteHeader(http.StatusBadRequest) + return + } + + // execute search for query provided, if no index for type send 404 + matches, err := db.SearchType(t, q) + if err == db.ErrNoSearchIndex { + res.WriteHeader(http.StatusNotFound) + return + } + if err != nil { + log.Println("[search] Error:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + // respond with json formatted results + bb, err := db.ContentMulti(matches) + if err != nil { + log.Println("[search] Error:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + // if we have matches, push the first as its matched by relevance + if len(bb) > 0 { + push(res, req, it, bb[0]) + } + + var result = []json.RawMessage{} + for i := range bb { + result = append(result, bb[i]) + } + + j, err := fmtJSON(result...) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + j, err = omit(it(), j) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + sendData(res, req, j) +} diff --git a/system/api/server.go b/system/api/server.go index c568877..209ddaa 100644 --- a/system/api/server.go +++ b/system/api/server.go @@ -13,4 +13,6 @@ func Run() { http.HandleFunc("/api/content/update", Record(CORS(updateContentHandler))) http.HandleFunc("/api/content/delete", Record(CORS(deleteContentHandler))) + + http.HandleFunc("/api/search", Record(CORS(searchContentHandler))) } diff --git a/system/db/content.go b/system/db/content.go index 97ea9b4..5db2896 100644 --- a/system/db/content.go +++ b/system/db/content.go @@ -121,6 +121,15 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error return 0, err } + go func() { + // update data in search index + target := fmt.Sprintf("%s:%s", ns, id) + err = UpdateSearchIndex(target, j) + if err != nil { + log.Println("[search] UpdateSearchIndex Error:", err) + } + }() + return cid, nil } @@ -128,7 +137,7 @@ func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, erro var j []byte t, ok := item.Types[ns] if !ok { - return nil, fmt.Errorf("namespace type not found:", ns) + return nil, fmt.Errorf("Namespace type not found: %s", ns) } // Unmarsal the existing values @@ -169,6 +178,8 @@ func insert(ns string, data url.Values) (int, error) { specifier = "__" + spec[1] } + var j []byte + var cid string err := store.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier)) if err != nil { @@ -181,7 +192,7 @@ func insert(ns string, data url.Values) (int, error) { if err != nil { return err } - cid := strconv.FormatUint(id, 10) + cid = strconv.FormatUint(id, 10) effectedID, err = strconv.Atoi(cid) if err != nil { return err @@ -197,7 +208,7 @@ func insert(ns string, data url.Values) (int, error) { data.Set("__specifier", specifier) } - j, err := postToJSON(ns, data) + j, err = postToJSON(ns, data) if err != nil { return err } @@ -238,6 +249,15 @@ func insert(ns string, data url.Values) (int, error) { return 0, err } + go func() { + // add data to seach index + target := fmt.Sprintf("%s:%s", ns, cid) + err = UpdateSearchIndex(target, j) + if err != nil { + log.Println("[search] UpdateSearchIndex Error:", err) + } + }() + return effectedID, nil } @@ -297,6 +317,17 @@ func DeleteContent(target string) error { return err } + go func() { + // delete indexed data from search index + if !strings.Contains(ns, "__") { + target = fmt.Sprintf("%s:%s", ns, id) + err = DeleteSearchIndex(target) + if err != nil { + log.Println("[search] DeleteSearchIndex Error:", err) + } + } + }() + // exception to typical "run in goroutine" pattern: // we want to have an updated admin view as soon as this is deleted, so // in some cases, the delete and redirect is faster than the sort, @@ -334,6 +365,23 @@ func Content(target string) ([]byte, error) { return val.Bytes(), nil } +// ContentMulti returns a set of content based on the the targets / identifiers +// provided in Ponzu target string format: Type:ID +// NOTE: All targets should be of the same type +func ContentMulti(targets []string) ([][]byte, error) { + var contents [][]byte + for i := range targets { + b, err := Content(targets[i]) + if err != nil { + return nil, err + } + + contents = append(contents, b) + } + + return contents, nil +} + // ContentBySlug does a lookup in the content index to find the type and id of // the requested content. Subsequently, issues the lookup in the type bucket and // returns the the type and data at that ID or nil if nothing exists. diff --git a/system/db/init.go b/system/db/init.go index 9125d3b..4e9c3cf 100644 --- a/system/db/init.go +++ b/system/db/init.go @@ -80,6 +80,12 @@ func Init() { go func() { for t := range item.Types { + err := MapSearchIndex(t) + if err != nil { + log.Fatalln(err) + return + } + SortContent(t) } }() diff --git a/system/db/search.go b/system/db/search.go new file mode 100644 index 0000000..b3d5fa1 --- /dev/null +++ b/system/db/search.go @@ -0,0 +1,162 @@ +package db + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ponzu-cms/ponzu/system/item" + + "encoding/json" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" +) + +var ( + // Search tracks all search indices to use throughout system + Search map[string]bleve.Index + + // ErrNoSearchIndex is for failed checks for an index in Search map + ErrNoSearchIndex = errors.New("No search index found for type provided") +) + +// Searchable ... +type Searchable interface { + SearchMapping() (*mapping.IndexMappingImpl, error) + IndexContent() bool +} + +func init() { + Search = make(map[string]bleve.Index) +} + +// MapSearchIndex creates the mapping for a type and tracks the index to be used within +// the system for adding/deleting/checking data +func MapSearchIndex(typeName string) error { + // type assert for Searchable, get configuration (which can be overridden) + // by Ponzu user if defines own SearchMapping() + it, ok := item.Types[typeName] + if !ok { + return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) + } + s, ok := it().(Searchable) + if !ok { + return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName) + } + + // skip setting or using index for types that shouldn't be indexed + if !s.IndexContent() { + return nil + } + + mapping, err := s.SearchMapping() + if err != nil { + return err + } + + idxName := typeName + ".index" + var idx bleve.Index + + // check if index exists, use it or create new one + pwd, err := os.Getwd() + if err != nil { + return err + } + + searchPath := filepath.Join(pwd, "search") + + err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) + if err != nil { + return err + } + + idxPath := filepath.Join(searchPath, idxName) + if _, err = os.Stat(idxPath); os.IsNotExist(err) { + idx, err = bleve.New(idxPath, mapping) + if err != nil { + return err + } + idx.SetName(idxName) + } else { + idx, err = bleve.Open(idxPath) + if err != nil { + return err + } + } + + // add the type name to the index and track the index + Search[typeName] = idx + + return nil +} + +// UpdateSearchIndex sets data into a content type's search index at the given +// identifier +func UpdateSearchIndex(id string, data interface{}) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // unmarshal json to struct, error if not registered + it, ok := item.Types[ns] + if !ok { + return fmt.Errorf("[search] UpdateSearchIndex Error: type '%s' doesn't exist", ns) + } + + p := it() + err := json.Unmarshal(data.([]byte), &p) + if err != nil { + return err + } + + // add data to search index + return idx.Index(id, p) + } + + return nil +} + +// DeleteSearchIndex removes data from a content type's search index at the +// given identifier +func DeleteSearchIndex(id string) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := Search[ns] + if ok { + // add data to search index + return idx.Delete(id) + } + + return nil +} + +// SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs, +// and an error. If there is no search index for the typeName (Type) provided, +// db.ErrNoSearchIndex will be returned as the error +func SearchType(typeName, query string) ([]string, error) { + idx, ok := Search[typeName] + if !ok { + return nil, ErrNoSearchIndex + } + + q := bleve.NewQueryStringQuery(query) + req := bleve.NewSearchRequest(q) + res, err := idx.Search(req) + if err != nil { + return nil, err + } + + var results []string + for _, hit := range res.Hits { + results = append(results, hit.ID) + } + + return results, nil +} diff --git a/system/item/item.go b/system/item/item.go index 99d70a8..8f392ca 100644 --- a/system/item/item.go +++ b/system/item/item.go @@ -7,6 +7,8 @@ import ( "strings" "unicode" + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" uuid "github.com/satori/go.uuid" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" @@ -208,6 +210,21 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error { return nil } +// SearchMapping returns a default implementation of a Bleve IndexMappingImpl +// partially implements db.Searchable +func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) { + mapping := bleve.NewIndexMapping() + mapping.StoreDynamic = false + + return mapping, nil +} + +// IndexContent determines if a type should be indexed for searching +// partially implements db.Searchable +func (i Item) IndexContent() bool { + return false +} + // Slug returns a URL friendly string from the title of a post item func Slug(i Identifiable) (string, error) { // get the name of the post item |