quorum/swarm/api/http/server.go

770 lines
22 KiB
Go

// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
A simple http server interface to Swarm
*/
package http
import (
"archive/tar"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/rs/cors"
)
//setup metrics
var (
postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil)
postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil)
postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil)
postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil)
deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil)
deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil)
getCount = metrics.NewRegisteredCounter("api.http.get.count", nil)
getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil)
getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil)
getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil)
getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
requestCount = metrics.NewRegisteredCounter("http.request.count", nil)
htmlRequestCount = metrics.NewRegisteredCounter("http.request.html.count", nil)
jsonRequestCount = metrics.NewRegisteredCounter("http.request.json.count", nil)
requestTimer = metrics.NewRegisteredResettingTimer("http.request.time", nil)
)
// ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings.
type ServerConfig struct {
Addr string
CorsString string
}
// browser API for registering bzz url scheme handlers:
// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
// electron (chromium) api for registering bzz url scheme handlers:
// https://github.com/atom/electron/blob/master/docs/api/protocol.md
// starts up http server
func StartHttpServer(api *api.Api, config *ServerConfig) {
var allowedOrigins []string
for _, domain := range strings.Split(config.CorsString, ",") {
allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
}
c := cors.New(cors.Options{
AllowedOrigins: allowedOrigins,
AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"},
MaxAge: 600,
AllowedHeaders: []string{"*"},
})
hdlr := c.Handler(NewServer(api))
go http.ListenAndServe(config.Addr, hdlr)
}
func NewServer(api *api.Api) *Server {
return &Server{api}
}
type Server struct {
api *api.Api
}
// Request wraps http.Request and also includes the parsed bzz URI
type Request struct {
http.Request
uri *api.URI
}
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
postRawCount.Inc(1)
if r.uri.Path != "" {
postRawFail.Inc(1)
s.BadRequest(w, r, "raw POST request cannot contain a path")
return
}
if r.Header.Get("Content-Length") == "" {
postRawFail.Inc(1)
s.BadRequest(w, r, "missing Content-Length header in request")
return
}
key, err := s.api.Store(r.Body, r.ContentLength, nil)
if err != nil {
postRawFail.Inc(1)
s.Error(w, r, err)
return
}
s.logDebug("content for %s stored", key.Log())
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, key)
}
// HandlePostFiles handles a POST request (or deprecated PUT request) to
// bzz:/<hash>/<path> which contains either a single file or multiple files
// (either a tar archive or multipart form), adds those files either to an
// existing manifest or to a new manifest under <path> and returns the
// resulting manifest hash as a text/plain response
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
postFilesCount.Inc(1)
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
postFilesFail.Inc(1)
s.BadRequest(w, r, err.Error())
return
}
var key storage.Key
if r.uri.Addr != "" {
key, err = s.api.Resolve(r.uri)
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
} else {
key, err = s.api.NewManifest()
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, err)
return
}
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
switch contentType {
case "application/x-tar":
return s.handleTarUpload(r, mw)
case "multipart/form-data":
return s.handleMultipartUpload(r, params["boundary"], mw)
default:
return s.handleDirectUpload(r, mw)
}
})
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
tr := tar.NewReader(req.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading tar stream: %s", err)
}
// only store regular files
if !hdr.FileInfo().Mode().IsRegular() {
continue
}
// add the entry under the path from the request
path := path.Join(req.uri.Path, hdr.Name)
entry := &api.ManifestEntry{
Path: path,
ContentType: hdr.Xattrs["user.swarm.content-type"],
Mode: hdr.Mode,
Size: hdr.Size,
ModTime: hdr.ModTime,
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(tr, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
mr := multipart.NewReader(req.Body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading multipart form: %s", err)
}
var size int64
var reader io.Reader = part
if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return fmt.Errorf("error parsing multipart content length: %s", err)
}
reader = part
} else {
// copy the part to a tmp file to get its size
tmp, err := ioutil.TempFile("", "swarm-multipart")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
size, err = io.Copy(tmp, part)
if err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
reader = tmp
}
// add the entry under the path from the request
name := part.FileName()
if name == "" {
name = part.FormName()
}
path := path.Join(req.uri.Path, name)
entry := &api.ManifestEntry{
Path: path,
ContentType: part.Header.Get("Content-Type"),
Size: size,
ModTime: time.Now(),
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
Path: req.uri.Path,
ContentType: req.Header.Get("Content-Type"),
Mode: 0644,
Size: req.ContentLength,
ModTime: time.Now(),
})
if err != nil {
return err
}
s.logDebug("content for %s stored", key.Log())
return nil
}
// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
// <path> from <manifest> and returns the resulting manifest hash as a
// text/plain response
func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
deleteCount.Inc(1)
key, err := s.api.Resolve(r.uri)
if err != nil {
deleteFail.Inc(1)
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
return mw.RemoveEntry(r.uri.Path)
})
if err != nil {
deleteFail.Inc(1)
s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
// HandleGet handles a GET request to
// - bzz-raw://<key> and responds with the raw content stored at the
// given storage key
// - bzz-hash://<key> and responds with the hash of the content stored
// at the given storage key as a text/plain response
func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
getCount.Inc(1)
key, err := s.api.Resolve(r.uri)
if err != nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
if r.uri.Path != "" {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
getFail.Inc(1)
s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
return
}
var entry *api.ManifestEntry
walker.Walk(func(e *api.ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == r.uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
}
// ignore non-manifest files
if e.ContentType != api.ManifestType {
return nil
}
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(r.uri.Path, e.Path) {
return nil
}
return api.SkipManifest
})
if entry == nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded"))
return
}
key = storage.Key(common.Hex2Bytes(entry.Hash))
}
// check the root chunk exists by retrieving the file's size
reader := s.api.Retrieve(key)
if _, err := reader.Size(nil); err != nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err))
return
}
switch {
case r.uri.Raw() || r.uri.DeprecatedRaw():
// allow the request to overwrite the content type using a query
// parameter
contentType := "application/octet-stream"
if typ := r.URL.Query().Get("content_type"); typ != "" {
contentType = typ
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
case r.uri.Hash():
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, key)
}
}
// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
// header of "application/x-tar" and returns a tar stream of all files
// contained in the manifest
func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
getFilesCount.Inc(1)
if r.uri.Path != "" {
getFilesFail.Inc(1)
s.BadRequest(w, r, "files request cannot contain a path")
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getFilesFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
getFilesFail.Inc(1)
s.Error(w, r, err)
return
}
tw := tar.NewWriter(w)
defer tw.Close()
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
err = walker.Walk(func(entry *api.ManifestEntry) error {
// ignore manifests (walk will recurse into them)
if entry.ContentType == api.ManifestType {
return nil
}
// retrieve the entry's key and size
reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
size, err := reader.Size(nil)
if err != nil {
return err
}
// write a tar header for the entry
hdr := &tar.Header{
Name: entry.Path,
Mode: entry.Mode,
Size: size,
ModTime: entry.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": entry.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
// copy the file into the tar stream
n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
if err != nil {
return err
} else if n != size {
return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
}
return nil
})
if err != nil {
getFilesFail.Inc(1)
s.logError("error generating tar stream: %s", err)
}
}
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
getListCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getListFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
getListFail.Inc(1)
s.Error(w, r, err)
return
}
// if the client wants HTML (e.g. a browser) then render the list as a
// HTML index with relative URLs
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
URI: &api.URI{
Scheme: "bzz",
Addr: r.uri.Addr,
Path: r.uri.Path,
},
List: &list,
})
if err != nil {
getListFail.Inc(1)
s.logError("error rendering list HTML: %s", err)
}
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(&list)
}
func (s *Server) getManifestList(key storage.Key, prefix string) (list api.ManifestList, err error) {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
return
}
err = walker.Walk(func(entry *api.ManifestEntry) error {
// handle non-manifest files
if entry.ContentType != api.ManifestType {
// ignore the file if it doesn't have the specified prefix
if !strings.HasPrefix(entry.Path, prefix) {
return nil
}
// if the path after the prefix contains a slash, add a
// common prefix to the list, otherwise add the entry
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return nil
}
if entry.Path == "" {
entry.Path = "/"
}
list.Entries = append(list.Entries, entry)
return nil
}
// if the manifest's path is a prefix of the specified prefix
// then just recurse into the manifest by returning nil and
// continuing the walk
if strings.HasPrefix(prefix, entry.Path) {
return nil
}
// if the manifest's path has the specified prefix, then if the
// path after the prefix contains a slash, add a common prefix
// to the list and skip the manifest, otherwise recurse into
// the manifest by returning nil and continuing the walk
if strings.HasPrefix(entry.Path, prefix) {
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return api.SkipManifest
}
return nil
}
// the manifest neither has the prefix or needs recursing in to
// so just skip it
return api.SkipManifest
})
return list, nil
}
// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
// with the content of the file at <path> from the given <manifest>
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
getFileCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getFileFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
reader, contentType, status, err := s.api.Get(key, r.uri.Path)
if err != nil {
switch status {
case http.StatusNotFound:
getFileNotFound.Inc(1)
s.NotFound(w, r, err)
default:
getFileFail.Inc(1)
s.Error(w, r, err)
}
return
}
//the request results in ambiguous files
//e.g. /read with readme.md and readinglist.txt available in manifest
if status == http.StatusMultipleChoices {
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
getFileFail.Inc(1)
s.Error(w, r, err)
return
}
s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list))
//show a nice page links to available entries
ShowMultipleChoices(w, r, list)
return
}
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(nil); err != nil {
getFileNotFound.Inc(1)
s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err))
return
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if metrics.Enabled {
//The increment for request count and request timer themselves have a flag check
//for metrics.Enabled. Nevertheless, we introduce the if here because we
//are looking into the header just to see what request type it is (json/html).
//So let's take advantage and add all metrics related stuff here
requestCount.Inc(1)
defer requestTimer.UpdateSince(time.Now())
if r.Header.Get("Accept") == "application/json" {
jsonRequestCount.Inc(1)
} else {
htmlRequestCount.Inc(1)
}
}
s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") {
err := landingPageTemplate.Execute(w, nil)
if err != nil {
s.logError("error rendering landing page: %s", err)
}
return
}
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
req := &Request{Request: *r, uri: uri}
if err != nil {
s.logError("Invalid URI %q: %s", r.URL.Path, err)
s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err))
return
}
s.logDebug("%s request received for %s", r.Method, uri)
switch r.Method {
case "POST":
if uri.Raw() || uri.DeprecatedRaw() {
s.HandlePostRaw(w, req)
} else {
s.HandlePostFiles(w, req)
}
case "PUT":
// DEPRECATED:
// clients should send a POST request (the request creates a
// new manifest leaving the existing one intact, so it isn't
// strictly a traditional PUT request which replaces content
// at a URI, and POST is more ubiquitous)
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, req, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
} else {
s.HandlePostFiles(w, req)
}
case "DELETE":
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, req, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
return
}
s.HandleDelete(w, req)
case "GET":
if uri.Raw() || uri.Hash() || uri.DeprecatedRaw() {
s.HandleGet(w, req)
return
}
if uri.List() {
s.HandleGetList(w, req)
return
}
if r.Header.Get("Accept") == "application/x-tar" {
s.HandleGetFiles(w, req)
return
}
s.HandleGetFile(w, req)
default:
ShowError(w, req, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed)
}
}
func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
mw, err := s.api.NewManifestWriter(key, nil)
if err != nil {
return nil, err
}
if err := update(mw); err != nil {
return nil, err
}
key, err = mw.Store()
if err != nil {
return nil, err
}
s.logDebug("generated manifest %s", key)
return key, nil
}
func (s *Server) logDebug(format string, v ...interface{}) {
log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) logError(format string, v ...interface{}) {
log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Request.Method, r.uri, reason), http.StatusBadRequest)
}
func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
ShowError(w, r, fmt.Sprintf("Error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusInternalServerError)
}
func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) {
ShowError(w, r, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusNotFound)
}