From 8f12938f28c0637a9261c7fcb4b0e311b8ba3b40 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 03:49:49 -0700 Subject: adding csv format interface and handler impl --- system/admin/export.go | 133 +++++++++++++++++++++++++++++++++++++++++++++++ system/admin/handlers.go | 49 +++++++++++++++-- system/admin/server.go | 1 + 3 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 system/admin/export.go (limited to 'system') diff --git a/system/admin/export.go b/system/admin/export.go new file mode 100644 index 0000000..82b5279 --- /dev/null +++ b/system/admin/export.go @@ -0,0 +1,133 @@ +package admin + +import ( + "fmt" + "log" + "net/http" + "path/filepath" + "strings" + "time" + + "encoding/csv" + + "io/ioutil" + "os" + + "github.com/ponzu-cms/ponzu/management/format" + "github.com/ponzu-cms/ponzu/system/db" + "github.com/ponzu-cms/ponzu/system/item" + "github.com/tidwall/gjson" +) + +func exportHandler(res http.ResponseWriter, req *http.Request) { + // /admin/contents/export?type=Blogpost&format=csv + q := req.URL.Query() + t := q.Get("type") + f := strings.ToLower(q.Get("format")) + + if t == "" || f == "" { + v, err := Error400() + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + res.WriteHeader(http.StatusBadRequest) + _, err = res.Write(v) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + + } + + pt, ok := item.Types[t] + if !ok { + res.WriteHeader(http.StatusBadRequest) + return + } + + switch f { + case "csv": + csv, ok := pt().(format.CSVFormattable) + if !ok { + res.WriteHeader(http.StatusBadRequest) + return + } + + fields := csv.FormatCSV() + exportCSV(res, req, pt, fields) + + default: + res.WriteHeader(http.StatusBadRequest) + return + } +} + +func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{}, fields []string) { + tmpFile, err := ioutil.TempFile(os.TempDir(), "exportcsv-") + if err != nil { + log.Println("Failed to create tmp file for CSV export:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + defer os.Remove(filepath.Join(os.TempDir(), tmpFile.Name())) + defer tmpFile.Close() + + csvBuf := csv.NewWriter(tmpFile) + + t := req.URL.Query().Get("type") + + // get content data and loop through creating a csv row per result + bb := db.ContentAll(t) + + err = csvBuf.Write(fields) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + log.Println("Failed to write column headers:", fields) + return + } + + for row := range bb { + // unmarshal data and loop over fields + rowBuf := []string{} + + for _, col := range fields { + // pull out each field as the column value + result := gjson.GetBytes(bb[row], col) + + // append it to the buffer + rowBuf = append(rowBuf, result.String()) + } + + // write row to csv + err := csvBuf.Write(rowBuf) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + log.Println("Failed to write column headers:", fields) + return + } + } + + // write the buffer to a content-disposition response + csvB, err := ioutil.ReadAll(tmpFile) + if err != nil { + log.Println("Failed to read tmp file for CSV export:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + ts := time.Now().Unix() + disposition := `attachment; filename="export-%s-%d.csv"` + + res.Header().Set("Content-Type", "text/csv") + res.Header().Set("Content-Disposition", fmt.Sprintf(disposition, t, ts)) + res.Header().Set("Content-Length", fmt.Sprintf("%d", len(csvB))) + + _, err = res.Write(csvB) + if err != nil { + log.Println("Failed to write tmp file to response for CSV export:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } +} diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 23c7e50..2a76e58 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ponzu-cms/ponzu/management/editor" + "github.com/ponzu-cms/ponzu/management/format" "github.com/ponzu-cms/ponzu/management/manager" "github.com/ponzu-cms/ponzu/system/addon" "github.com/ponzu-cms/ponzu/system/admin/config" @@ -1492,8 +1493,24 @@ func contentsHandler(res http.ResponseWriter, req *http.Request) { ` - btn := `
New ` + t + `
` - html = html + b.String() + script + btn + btn := `
+ + New ` + t + ` + +
` + html = html + b.String() + btn + + if _, ok := pt.(format.CSVFormattable); ok { + btn = `
+ + system_update_alt + .CSV + +
` + html = html + b.String() + btn + } + + html += `` + script adminView, err := Admin([]byte(html)) if err != nil { @@ -2422,7 +2439,15 @@ func searchHandler(res http.ResponseWriter, req *http.Request) { posts := db.ContentAll(t + specifier) b := &bytes.Buffer{} - p := item.Types[t]().(editor.Editable) + pt, ok := item.Types[t] + if !ok { + res.WriteHeader(http.StatusBadRequest) + return + } + + post := pt() + + p := post.(editor.Editable) html := `
@@ -2499,9 +2524,25 @@ func searchHandler(res http.ResponseWriter, req *http.Request) { return } - btn := `
New ` + t + `
` + btn := `
+ + New ` + t + ` + +
` html = html + b.String() + btn + if _, ok := post.(format.CSVFormattable); ok { + btn = `
+ + system_update_alt + .CSV + +
` + html = html + b.String() + btn + } + + html += `
` + adminView, err := Admin([]byte(html)) if err != nil { log.Println(err) diff --git a/system/admin/server.go b/system/admin/server.go index 2180441..426dabd 100644 --- a/system/admin/server.go +++ b/system/admin/server.go @@ -37,6 +37,7 @@ func Run() { http.HandleFunc("/admin/contents", user.Auth(contentsHandler)) http.HandleFunc("/admin/contents/search", user.Auth(searchHandler)) + http.HandleFunc("/admin/contents/export", user.Auth(exportHandler)) http.HandleFunc("/admin/edit", user.Auth(editHandler)) http.HandleFunc("/admin/edit/delete", user.Auth(deleteHandler)) -- cgit v1.2.3 From 779631d5dc46afbc765ee138a79bab962f11312b Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 03:59:27 -0700 Subject: flush csv writer and fix ui --- system/admin/export.go | 11 ++++++----- system/admin/handlers.go | 24 ++++++++++-------------- 2 files changed, 16 insertions(+), 19 deletions(-) (limited to 'system') diff --git a/system/admin/export.go b/system/admin/export.go index 82b5279..cdd43b7 100644 --- a/system/admin/export.go +++ b/system/admin/export.go @@ -1,21 +1,20 @@ package admin import ( + "encoding/csv" "fmt" + "io/ioutil" "log" "net/http" + "os" "path/filepath" "strings" "time" - "encoding/csv" - - "io/ioutil" - "os" - "github.com/ponzu-cms/ponzu/management/format" "github.com/ponzu-cms/ponzu/system/db" "github.com/ponzu-cms/ponzu/system/item" + "github.com/tidwall/gjson" ) @@ -109,6 +108,8 @@ func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{} } } + csvBuf.Flush() + // write the buffer to a content-disposition response csvB, err := ioutil.ReadAll(tmpFile) if err != nil { diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 2a76e58..2d8b814 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -1496,21 +1496,19 @@ func contentsHandler(res http.ResponseWriter, req *http.Request) { btn := `
New ` + t + ` - -
` + ` html = html + b.String() + btn if _, ok := pt.(format.CSVFormattable); ok { - btn = `
- + btn = `
+ system_update_alt .CSV - -
` + ` html = html + b.String() + btn } - html += `` + script + html += `` + script adminView, err := Admin([]byte(html)) if err != nil { @@ -2527,21 +2525,19 @@ func searchHandler(res http.ResponseWriter, req *http.Request) { btn := `
New ` + t + ` - -
` + ` html = html + b.String() + btn if _, ok := post.(format.CSVFormattable); ok { - btn = `
- + btn = `
+ system_update_alt .CSV - -
` + ` html = html + b.String() + btn } - html += `` + html += `` adminView, err := Admin([]byte(html)) if err != nil { -- cgit v1.2.3 From c76b8e021091e08c49bd7ba28e1b1c8a7632f749 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 04:44:21 -0700 Subject: manually call remove and close, debug response --- system/admin/export.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'system') diff --git a/system/admin/export.go b/system/admin/export.go index cdd43b7..46a659b 100644 --- a/system/admin/export.go +++ b/system/admin/export.go @@ -7,7 +7,6 @@ import ( "log" "net/http" "os" - "path/filepath" "strings" "time" @@ -70,8 +69,6 @@ func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{} res.WriteHeader(http.StatusInternalServerError) return } - defer os.Remove(filepath.Join(os.TempDir(), tmpFile.Name())) - defer tmpFile.Close() csvBuf := csv.NewWriter(tmpFile) @@ -80,6 +77,7 @@ func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{} // get content data and loop through creating a csv row per result bb := db.ContentAll(t) + // add field names to first row err = csvBuf.Write(fields) if err != nil { res.WriteHeader(http.StatusInternalServerError) @@ -111,24 +109,29 @@ func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{} csvBuf.Flush() // write the buffer to a content-disposition response - csvB, err := ioutil.ReadAll(tmpFile) + fi, err := tmpFile.Stat() if err != nil { - log.Println("Failed to read tmp file for CSV export:", err) + log.Println("Failed to read tmp file info for CSV export:", err) res.WriteHeader(http.StatusInternalServerError) return } + err = tmpFile.Close() + if err != nil { + log.Println("Failed to close tmp file for CSV export:", err) + } + ts := time.Now().Unix() disposition := `attachment; filename="export-%s-%d.csv"` res.Header().Set("Content-Type", "text/csv") res.Header().Set("Content-Disposition", fmt.Sprintf(disposition, t, ts)) - res.Header().Set("Content-Length", fmt.Sprintf("%d", len(csvB))) + res.Header().Set("Content-Length", fmt.Sprintf("%d", int(fi.Size()))) - _, err = res.Write(csvB) + http.ServeFile(res, req, tmpFile.Name()) + + err = os.Remove(tmpFile.Name()) if err != nil { - log.Println("Failed to write tmp file to response for CSV export:", err) - res.WriteHeader(http.StatusInternalServerError) - return + log.Println("Failed to remove tmp file for CSV export:", err) } } -- cgit v1.2.3 From 5cf617fb2853004accbf40f5b2fd93f3ced4949a Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 09:37:51 -0700 Subject: changing file perm on temp --- system/admin/export.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'system') diff --git a/system/admin/export.go b/system/admin/export.go index 46a659b..2453eb8 100644 --- a/system/admin/export.go +++ b/system/admin/export.go @@ -70,6 +70,11 @@ func exportCSV(res http.ResponseWriter, req *http.Request, pt func() interface{} return } + err = os.Chmod(tmpFile.Name(), 0666) + if err != nil { + log.Println("chmod err:", err) + } + csvBuf := csv.NewWriter(tmpFile) t := req.URL.Query().Get("type") -- cgit v1.2.3 From c6a38a5178ab8661240c16983c9e1dcd08571abc Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 09:38:06 -0700 Subject: fixing UI --- system/admin/handlers.go | 6 ++---- system/admin/static/dashboard/css/admin.css | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'system') diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 2d8b814..0459b4f 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -1497,18 +1497,16 @@ func contentsHandler(res http.ResponseWriter, req *http.Request) { New ` + t + ` ` - html = html + b.String() + btn if _, ok := pt.(format.CSVFormattable); ok { - btn = `
+ btn += `
system_update_alt .CSV ` - html = html + b.String() + btn } - html += `` + script + html += b.String() + script + btn + `` adminView, err := Admin([]byte(html)) if err != nil { diff --git a/system/admin/static/dashboard/css/admin.css b/system/admin/static/dashboard/css/admin.css index 06df137..45ca5d8 100644 --- a/system/admin/static/dashboard/css/admin.css +++ b/system/admin/static/dashboard/css/admin.css @@ -81,7 +81,7 @@ li a:hover { transition: color 0.3s ease; } -a.new-post { +a.new-post, a.export-post { margin: 0.5rem 0 1rem 0.75rem; } -- cgit v1.2.3 From b0aacbefa498e144e05f41dba8903c0753af9829 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 11:32:26 -0700 Subject: update csv label --- system/admin/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system') diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 0459b4f..611955e 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -1502,7 +1502,7 @@ func contentsHandler(res http.ResponseWriter, req *http.Request) { btn += `
system_update_alt - .CSV + CSV ` } -- cgit v1.2.3 From 0ee16cc099c4fea518e9e57ff4b5166aba54ee34 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 15 May 2017 11:39:31 -0700 Subject: update csv label --- system/admin/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'system') diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 611955e..a950633 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -2530,7 +2530,7 @@ func searchHandler(res http.ResponseWriter, req *http.Request) { btn = `
system_update_alt - .CSV + CSV ` html = html + b.String() + btn } -- cgit v1.2.3