Merge pull request #116 from ponzu-cms/ponzu-dev

[core] Configurable full-text search with Bleve + search API endpoint

7 files changed, 326 insertions, 3 deletions

diff --git a/README.md b/README.md
index c79c6c6..0aa531a 100644
--- a/README.md
+++ b/README.md
@@ -295,6 +295,7 @@ $ ponzu --dev -fork=github.com/nilslice/ponzu new /path/to/new/project
 - [golang.org/x/text/transform](https://golang.org/x/text/transform)
 - [golang.org/x/crypto/bcrypt](https://golang.org/x/crypto/bcrypt)
 - [golang.org/x/net/http2](https://golang.org/x/net/http2)
+- [github.com/blevesearch/bleve](https://github.com/blevesearch/bleve)
 - [github.com/nilslice/jwt](https://github.com/nilslice/jwt)
 - [github.com/nilslice/email](https://github.com/nilslice/email)
 - [github.com/gorilla/schema](https://github.com/gorilla/schema)
diff --git a/system/api/search.go b/system/api/search.go
new file mode 100644
index 0000000..c743d9f
--- /dev/null
+++ b/system/api/search.go
@@ -0,0 +1,87 @@
+package api
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"net/url"
+
+	"github.com/ponzu-cms/ponzu/system/db"
+	"github.com/ponzu-cms/ponzu/system/item"
+)
+
+func searchContentHandler(res http.ResponseWriter, req *http.Request) {
+	qs := req.URL.Query()
+	t := qs.Get("type")
+	// type must be set, future version may compile multi-type result set
+	if t == "" {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	it, ok := item.Types[t]
+	if !ok {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if hide(it(), res, req) {
+		return
+	}
+
+	q, err := url.QueryUnescape(qs.Get("q"))
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// q must be set
+	if q == "" {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// execute search for query provided, if no index for type send 404
+	matches, err := db.SearchType(t, q)
+	if err == db.ErrNoSearchIndex {
+		res.WriteHeader(http.StatusNotFound)
+		return
+	}
+	if err != nil {
+		log.Println("[search] Error:", err)
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// respond with json formatted results
+	bb, err := db.ContentMulti(matches)
+	if err != nil {
+		log.Println("[search] Error:", err)
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// if we have matches, push the first as its matched by relevance
+	if len(bb) > 0 {
+		push(res, req, it, bb[0])
+	}
+
+	var result = []json.RawMessage{}
+	for i := range bb {
+		result = append(result, bb[i])
+	}
+
+	j, err := fmtJSON(result...)
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	j, err = omit(it(), j)
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	sendData(res, req, j)
+}
diff --git a/system/api/server.go b/system/api/server.go
index c568877..209ddaa 100644
--- a/system/api/server.go
+++ b/system/api/server.go
@@ -13,4 +13,6 @@ func Run() {
 	http.HandleFunc("/api/content/update", Record(CORS(updateContentHandler)))
 
 	http.HandleFunc("/api/content/delete", Record(CORS(deleteContentHandler)))
+
+	http.HandleFunc("/api/search", Record(CORS(searchContentHandler)))
 }
diff --git a/system/db/content.go b/system/db/content.go
index 97ea9b4..5db2896 100644
--- a/system/db/content.go
+++ b/system/db/content.go
@@ -121,6 +121,15 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error
 		return 0, err
 	}
 
+	go func() {
+		// update data in search index
+		target := fmt.Sprintf("%s:%s", ns, id)
+		err = UpdateSearchIndex(target, j)
+		if err != nil {
+			log.Println("[search] UpdateSearchIndex Error:", err)
+		}
+	}()
+
 	return cid, nil
 }
 
@@ -128,7 +137,7 @@ func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, erro
 	var j []byte
 	t, ok := item.Types[ns]
 	if !ok {
-		return nil, fmt.Errorf("namespace type not found:", ns)
+		return nil, fmt.Errorf("Namespace type not found: %s", ns)
 	}
 
 	// Unmarsal the existing values
@@ -169,6 +178,8 @@ func insert(ns string, data url.Values) (int, error) {
 		specifier = "__" + spec[1]
 	}
 
+	var j []byte
+	var cid string
 	err := store.Update(func(tx *bolt.Tx) error {
 		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
 		if err != nil {
@@ -181,7 +192,7 @@ func insert(ns string, data url.Values) (int, error) {
 		if err != nil {
 			return err
 		}
-		cid := strconv.FormatUint(id, 10)
+		cid = strconv.FormatUint(id, 10)
 		effectedID, err = strconv.Atoi(cid)
 		if err != nil {
 			return err
@@ -197,7 +208,7 @@ func insert(ns string, data url.Values) (int, error) {
 			data.Set("__specifier", specifier)
 		}
 
-		j, err := postToJSON(ns, data)
+		j, err = postToJSON(ns, data)
 		if err != nil {
 			return err
 		}
@@ -238,6 +249,15 @@ func insert(ns string, data url.Values) (int, error) {
 		return 0, err
 	}
 
+	go func() {
+		// add data to seach index
+		target := fmt.Sprintf("%s:%s", ns, cid)
+		err = UpdateSearchIndex(target, j)
+		if err != nil {
+			log.Println("[search] UpdateSearchIndex Error:", err)
+		}
+	}()
+
 	return effectedID, nil
 }
 
@@ -297,6 +317,17 @@ func DeleteContent(target string) error {
 		return err
 	}
 
+	go func() {
+		// delete indexed data from search index
+		if !strings.Contains(ns, "__") {
+			target = fmt.Sprintf("%s:%s", ns, id)
+			err = DeleteSearchIndex(target)
+			if err != nil {
+				log.Println("[search] DeleteSearchIndex Error:", err)
+			}
+		}
+	}()
+
 	// exception to typical "run in goroutine" pattern:
 	// we want to have an updated admin view as soon as this is deleted, so
 	// in some cases, the delete and redirect is faster than the sort,
@@ -334,6 +365,23 @@ func Content(target string) ([]byte, error) {
 	return val.Bytes(), nil
 }
 
+// ContentMulti returns a set of content based on the the targets / identifiers
+// provided in Ponzu target string format: Type:ID
+// NOTE: All targets should be of the same type
+func ContentMulti(targets []string) ([][]byte, error) {
+	var contents [][]byte
+	for i := range targets {
+		b, err := Content(targets[i])
+		if err != nil {
+			return nil, err
+		}
+
+		contents = append(contents, b)
+	}
+
+	return contents, nil
+}
+
 // ContentBySlug does a lookup in the content index to find the type and id of
 // the requested content. Subsequently, issues the lookup in the type bucket and
 // returns the the type and data at that ID or nil if nothing exists.
diff --git a/system/db/init.go b/system/db/init.go
index 9125d3b..4e9c3cf 100644
--- a/system/db/init.go
+++ b/system/db/init.go
@@ -80,6 +80,12 @@ func Init() {
 
 	go func() {
 		for t := range item.Types {
+			err := MapSearchIndex(t)
+			if err != nil {
+				log.Fatalln(err)
+				return
+			}
+
 			SortContent(t)
 		}
 	}()
diff --git a/system/db/search.go b/system/db/search.go
new file mode 100644
index 0000000..b3d5fa1
--- /dev/null
+++ b/system/db/search.go
@@ -0,0 +1,162 @@
+package db
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ponzu-cms/ponzu/system/item"
+
+	"encoding/json"
+
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+var (
+	// Search tracks all search indices to use throughout system
+	Search map[string]bleve.Index
+
+	// ErrNoSearchIndex is for failed checks for an index in Search map
+	ErrNoSearchIndex = errors.New("No search index found for type provided")
+)
+
+// Searchable ...
+type Searchable interface {
+	SearchMapping() (*mapping.IndexMappingImpl, error)
+	IndexContent() bool
+}
+
+func init() {
+	Search = make(map[string]bleve.Index)
+}
+
+// MapSearchIndex creates the mapping for a type and tracks the index to be used within
+// the system for adding/deleting/checking data
+func MapSearchIndex(typeName string) error {
+	// type assert for Searchable, get configuration (which can be overridden)
+	// by Ponzu user if defines own SearchMapping()
+	it, ok := item.Types[typeName]
+	if !ok {
+		return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName)
+	}
+	s, ok := it().(Searchable)
+	if !ok {
+		return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName)
+	}
+
+	// skip setting or using index for types that shouldn't be indexed
+	if !s.IndexContent() {
+		return nil
+	}
+
+	mapping, err := s.SearchMapping()
+	if err != nil {
+		return err
+	}
+
+	idxName := typeName + ".index"
+	var idx bleve.Index
+
+	// check if index exists, use it or create new one
+	pwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	searchPath := filepath.Join(pwd, "search")
+
+	err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	idxPath := filepath.Join(searchPath, idxName)
+	if _, err = os.Stat(idxPath); os.IsNotExist(err) {
+		idx, err = bleve.New(idxPath, mapping)
+		if err != nil {
+			return err
+		}
+		idx.SetName(idxName)
+	} else {
+		idx, err = bleve.Open(idxPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	// add the type name to the index and track the index
+	Search[typeName] = idx
+
+	return nil
+}
+
+// UpdateSearchIndex sets data into a content type's search index at the given
+// identifier
+func UpdateSearchIndex(id string, data interface{}) error {
+	// check if there is a search index to work with
+	target := strings.Split(id, ":")
+	ns := target[0]
+
+	idx, ok := Search[ns]
+	if ok {
+		// unmarshal json to struct, error if not registered
+		it, ok := item.Types[ns]
+		if !ok {
+			return fmt.Errorf("[search] UpdateSearchIndex Error: type '%s' doesn't exist", ns)
+		}
+
+		p := it()
+		err := json.Unmarshal(data.([]byte), &p)
+		if err != nil {
+			return err
+		}
+
+		// add data to search index
+		return idx.Index(id, p)
+	}
+
+	return nil
+}
+
+// DeleteSearchIndex removes data from a content type's search index at the
+// given identifier
+func DeleteSearchIndex(id string) error {
+	// check if there is a search index to work with
+	target := strings.Split(id, ":")
+	ns := target[0]
+
+	idx, ok := Search[ns]
+	if ok {
+		// add data to search index
+		return idx.Delete(id)
+	}
+
+	return nil
+}
+
+// SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs,
+// and an error. If there is no search index for the typeName (Type) provided,
+// db.ErrNoSearchIndex will be returned as the error
+func SearchType(typeName, query string) ([]string, error) {
+	idx, ok := Search[typeName]
+	if !ok {
+		return nil, ErrNoSearchIndex
+	}
+
+	q := bleve.NewQueryStringQuery(query)
+	req := bleve.NewSearchRequest(q)
+	res, err := idx.Search(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []string
+	for _, hit := range res.Hits {
+		results = append(results, hit.ID)
+	}
+
+	return results, nil
+}
diff --git a/system/item/item.go b/system/item/item.go
index 99d70a8..8f392ca 100644
--- a/system/item/item.go
+++ b/system/item/item.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
 	uuid "github.com/satori/go.uuid"
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
@@ -208,6 +210,21 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error {
 	return nil
 }
 
+// SearchMapping returns a default implementation of a Bleve IndexMappingImpl
+// partially implements db.Searchable
+func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) {
+	mapping := bleve.NewIndexMapping()
+	mapping.StoreDynamic = false
+
+	return mapping, nil
+}
+
+// IndexContent determines if a type should be indexed for searching
+// partially implements db.Searchable
+func (i Item) IndexContent() bool {
+	return false
+}
+
 // Slug returns a URL friendly string from the title of a post item
 func Slug(i Identifiable) (string, error) {
 	// get the name of the post item