Merge branch 'throttle-sort'

7 files changed, 345 insertions, 3 deletions

diff --git a/system/api/search.go b/system/api/search.go
new file mode 100644
index 0000000..ae6ac1c
--- /dev/null
+++ b/system/api/search.go
@@ -0,0 +1,82 @@
+package api
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"net/url"
+
+	"github.com/ponzu-cms/ponzu/system/db"
+	"github.com/ponzu-cms/ponzu/system/item"
+)
+
+func searchContentHandler(res http.ResponseWriter, req *http.Request) {
+	qs := req.URL.Query()
+	t := qs.Get("type")
+	// type must be set, future version may compile multi-type result set
+	if t == "" {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	it, ok := item.Types[t]
+	if !ok {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if hide(it(), res, req) {
+		return
+	}
+
+	q, err := url.QueryUnescape(qs.Get("q"))
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// q must be set
+	if q == "" {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// execute search for query provided, if no index for type send 404
+	matches, err := db.SearchType(t, q)
+	if err == db.ErrNoSearchIndex {
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	if err != nil {
+		log.Println("[search] Error:", err)
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// respond with json formatted results
+	bb, err := db.ContentMulti(matches)
+	if err != nil {
+		log.Println("[search] Error:", err)
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	var result = []json.RawMessage{}
+	for i := range bb {
+		result = append(result, bb[i])
+	}
+
+	j, err := fmtJSON(result...)
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	j, err = omit(it(), j)
+	if err != nil {
+		res.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	sendData(res, req, j)
+}
diff --git a/system/api/server.go b/system/api/server.go
index c568877..209ddaa 100644
--- a/system/api/server.go
+++ b/system/api/server.go
@@ -13,4 +13,6 @@ func Run() {
 	http.HandleFunc("/api/content/update", Record(CORS(updateContentHandler)))
 
 	http.HandleFunc("/api/content/delete", Record(CORS(deleteContentHandler)))
+
+	http.HandleFunc("/api/search", Record(CORS(searchContentHandler)))
 }
diff --git a/system/db/content.go b/system/db/content.go
index 97ea9b4..994860f 100644
--- a/system/db/content.go
+++ b/system/db/content.go
@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ponzu-cms/ponzu/system/item"
 
@@ -121,6 +122,15 @@ func update(ns, id string, data url.Values, existingContent *[]byte) (int, error
 		return 0, err
 	}
 
+	go func() {
+		// update data in search index
+		target := fmt.Sprintf("%s:%s", ns, id)
+		err = UpdateSearchIndex(target, string(j))
+		if err != nil {
+			log.Println("[search] UpdateSearchIndex Error:", err)
+		}
+	}()
+
 	return cid, nil
 }
 
@@ -128,7 +138,7 @@ func mergeData(ns string, data url.Values, existingContent []byte) ([]byte, erro
 	var j []byte
 	t, ok := item.Types[ns]
 	if !ok {
-		return nil, fmt.Errorf("namespace type not found:", ns)
+		return nil, fmt.Errorf("Namespace type not found: %s", ns)
 	}
 
 	// Unmarsal the existing values
@@ -169,6 +179,8 @@ func insert(ns string, data url.Values) (int, error) {
 		specifier = "__" + spec[1]
 	}
 
+	var j []byte
+	var cid string
 	err := store.Update(func(tx *bolt.Tx) error {
 		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
 		if err != nil {
@@ -181,7 +193,7 @@ func insert(ns string, data url.Values) (int, error) {
 		if err != nil {
 			return err
 		}
-		cid := strconv.FormatUint(id, 10)
+		cid = strconv.FormatUint(id, 10)
 		effectedID, err = strconv.Atoi(cid)
 		if err != nil {
 			return err
@@ -197,7 +209,7 @@ func insert(ns string, data url.Values) (int, error) {
 			data.Set("__specifier", specifier)
 		}
 
-		j, err := postToJSON(ns, data)
+		j, err = postToJSON(ns, data)
 		if err != nil {
 			return err
 		}
@@ -238,6 +250,15 @@ func insert(ns string, data url.Values) (int, error) {
 		return 0, err
 	}
 
+	go func() {
+		// add data to seach index
+		target := fmt.Sprintf("%s:%s", ns, cid)
+		err = UpdateSearchIndex(target, string(j))
+		if err != nil {
+			log.Println("[search] UpdateSearchIndex Error:", err)
+		}
+	}()
+
 	return effectedID, nil
 }
 
@@ -297,6 +318,17 @@ func DeleteContent(target string) error {
 		return err
 	}
 
+	go func() {
+		// delete indexed data from search index
+		if !strings.Contains(ns, "__") {
+			target = fmt.Sprintf("%s:%s", ns, id)
+			err = DeleteSearchIndex(target)
+			if err != nil {
+				log.Println("[search] DeleteSearchIndex Error:", err)
+			}
+		}
+	}()
+
 	// exception to typical "run in goroutine" pattern:
 	// we want to have an updated admin view as soon as this is deleted, so
 	// in some cases, the delete and redirect is faster than the sort,
@@ -334,6 +366,23 @@ func Content(target string) ([]byte, error) {
 	return val.Bytes(), nil
 }
 
+// ContentMulti returns a set of content based on the the targets / identifiers
+// provided in Ponzu target string format: Type:ID
+// NOTE: All targets should be of the same type
+func ContentMulti(targets []string) ([][]byte, error) {
+	var contents [][]byte
+	for i := range targets {
+		b, err := Content(targets[i])
+		if err != nil {
+			return nil, err
+		}
+
+		contents = append(contents, b)
+	}
+
+	return contents, nil
+}
+
 // ContentBySlug does a lookup in the content index to find the type and id of
 // the requested content. Subsequently, issues the lookup in the type bucket and
 // returns the the type and data at that ID or nil if nothing exists.
@@ -515,10 +564,54 @@ func Query(namespace string, opts QueryOptions) (int, [][]byte) {
 	return total, posts
 }
 
+var sortContentCalls = make(map[string]time.Time)
+var waitDuration = time.Millisecond * 4000
+
+func enoughTime(key string, withDelay bool) bool {
+	last, ok := sortContentCalls[key]
+	if !ok {
+		// no envocation yet
+		// track next evocation
+		sortContentCalls[key] = time.Now()
+		return true
+	}
+
+	// if our required wait time has not been met, return false
+	if !time.Now().After(last.Add(waitDuration)) {
+		return false
+	}
+
+	// dispatch a delayed envocation in case no additional one follows
+	if withDelay {
+		go func() {
+			select {
+			case <-time.After(waitDuration):
+				if enoughTime(key, false) {
+					// track next evocation
+					sortContentCalls[key] = time.Now()
+					SortContent(key)
+				} else {
+					// retrigger
+					SortContent(key)
+				}
+			}
+		}()
+	}
+
+	// track next evocation
+	sortContentCalls[key] = time.Now()
+	return true
+}
+
 // SortContent sorts all content of the type supplied as the namespace by time,
 // in descending order, from most recent to least recent
 // Should be called from a goroutine after SetContent is successful
 func SortContent(namespace string) {
+	// wait if running too frequently per namespace
+	if !enoughTime(namespace, true) {
+		return
+	}
+
 	// only sort main content types i.e. Post
 	if strings.Contains(namespace, "__") {
 		return
diff --git a/system/db/init.go b/system/db/init.go
index 9125d3b..4e9c3cf 100644
--- a/system/db/init.go
+++ b/system/db/init.go
@@ -80,6 +80,12 @@ func Init() {
 
 	go func() {
 		for t := range item.Types {
+			err := MapSearchIndex(t)
+			if err != nil {
+				log.Fatalln(err)
+				return
+			}
+
 			SortContent(t)
 		}
 	}()
diff --git a/system/db/search.go b/system/db/search.go
new file mode 100644
index 0000000..3e7a9d6
--- /dev/null
+++ b/system/db/search.go
@@ -0,0 +1,145 @@
+package db
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ponzu-cms/ponzu/system/item"
+
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+var (
+	// Search tracks all search indices to use throughout system
+	Search map[string]bleve.Index
+
+	// ErrNoSearchIndex is for failed checks for an index in Search map
+	ErrNoSearchIndex = errors.New("No search index found for type provided")
+)
+
+// Searchable ...
+type Searchable interface {
+	SearchMapping() (*mapping.IndexMappingImpl, error)
+}
+
+func init() {
+	Search = make(map[string]bleve.Index)
+}
+
+// MapSearchIndex creates the mapping for a type and tracks the index to be used within
+// the system for adding/deleting/checking data
+func MapSearchIndex(typeName string) error {
+	// type assert for Searchable, get configuration (which can be overridden)
+	// by Ponzu user if defines own SearchMapping()
+	it, ok := item.Types[typeName]
+	if !ok {
+		return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName)
+	}
+	s, ok := it().(Searchable)
+	if !ok {
+		return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName)
+	}
+
+	mapping, err := s.SearchMapping()
+	if err == item.ErrNoSearchMapping {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	idxName := typeName + ".index"
+	var idx bleve.Index
+
+	// check if index exists, use it or create new one
+	pwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	searchPath := filepath.Join(pwd, "search")
+
+	err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	idxPath := filepath.Join(searchPath, idxName)
+	if _, err = os.Stat(idxPath); os.IsNotExist(err) {
+		idx, err = bleve.New(idxPath, mapping)
+		if err != nil {
+			return err
+		}
+		idx.SetName(idxName)
+	} else {
+		idx, err = bleve.Open(idxPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	// add the type name to the index and track the index
+	Search[typeName] = idx
+
+	return nil
+}
+
+// UpdateSearchIndex sets data into a content type's search index at the given
+// identifier
+func UpdateSearchIndex(id string, data interface{}) error {
+	// check if there is a search index to work with
+	target := strings.Split(id, ":")
+	ns := target[0]
+
+	idx, ok := Search[ns]
+	if ok {
+		// add data to search index
+		return idx.Index(id, data)
+	}
+
+	return nil
+}
+
+// DeleteSearchIndex removes data from a content type's search index at the
+// given identifier
+func DeleteSearchIndex(id string) error {
+	// check if there is a search index to work with
+	target := strings.Split(id, ":")
+	ns := target[0]
+
+	idx, ok := Search[ns]
+	if ok {
+		// add data to search index
+		return idx.Delete(id)
+	}
+
+	return nil
+}
+
+// SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs,
+// and an error. If there is no search index for the typeName (Type) provided,
+// db.ErrNoSearchIndex will be returned as the error
+func SearchType(typeName, query string) ([]string, error) {
+	idx, ok := Search[typeName]
+	if !ok {
+		return nil, ErrNoSearchIndex
+	}
+
+	q := bleve.NewQueryStringQuery(query)
+	req := bleve.NewSearchRequest(q)
+	res, err := idx.Search(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []string
+	for _, hit := range res.Hits {
+		results = append(results, hit.ID)
+	}
+
+	return results, nil
+}
diff --git a/system/item/item.go b/system/item/item.go
index 99d70a8..4750ef5 100644
--- a/system/item/item.go
+++ b/system/item/item.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
 	uuid "github.com/satori/go.uuid"
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
@@ -208,6 +210,15 @@ func (i Item) AfterReject(res http.ResponseWriter, req *http.Request) error {
 	return nil
 }
 
+// SearchMapping returns a default implementation of a Bleve IndexMappingImpl
+// partially implements db.Searchable
+func (i Item) SearchMapping() (*mapping.IndexMappingImpl, error) {
+	mapping := bleve.NewIndexMapping()
+	mapping.StoreDynamic = false
+
+	return mapping, nil
+}
+
 // Slug returns a URL friendly string from the title of a post item
 func Slug(i Identifiable) (string, error) {
 	// get the name of the post item
diff --git a/system/item/types.go b/system/item/types.go
index bcae58a..dbf13af 100644
--- a/system/item/types.go
+++ b/system/item/types.go
@@ -26,6 +26,9 @@ var (
 	// if requested by a valid admin or user
 	ErrAllowHiddenItem = errors.New(`Allow hidden item`)
 
+	// ErrNoSearchMapping can be used to tell the system not to create an index mapping
+	ErrNoSearchMapping = errors.New(`No search mapping for item`)
+
 	// Types is a map used to reference a type name to its actual Editable type
 	// mainly for lookups in /admin route based utilities
 	Types map[string]func() interface{}