poanetwork
/
quorum
mirror of https://github.com/poanetwork/quorum.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

swarm/api: refactor and improve HTTP API (#3773) 

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).
This commit is contained in:
Lewis Marshall 2017-04-06 23:22:22 +01:00 committed by Felix Lange
parent 9aca9e6deb
commit 71fdaa4238
20 changed files with 1822 additions and 706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/swarm/list.go
View File

@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
entries, err := client.ManifestFileList(manifest, prefix)
list, err := client.List(manifest, prefix)
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
@ -52,7 +52,10 @@ func list(ctx *cli.Context) {
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
for _, entry := range entries {
for _, prefix := range list.CommonPrefixes {
fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
}
for _, entry := range list.Entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}

63
cmd/swarm/manifest.go
View File

@ -25,6 +25,7 @@ import (
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@ -42,7 +43,7 @@ func add(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot swarm.Manifest
mroot api.Manifest
)
if len(args) > 3 {
@ -76,7 +77,7 @@ func update(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot swarm.Manifest
mroot api.Manifest
)
if len(args) > 3 {
ctype = args[3]
@ -106,7 +107,7 @@ func remove(ctx *cli.Context) {
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot swarm.Manifest
mroot api.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
@ -125,11 +126,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@ -163,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := swarm.Manifest{}
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -173,9 +170,9 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
newEntry := swarm.ManifestEntry{
Path: path,
newEntry := api.ManifestEntry{
Hash: hash,
Path: path,
ContentType: ctype,
}
mroot.Entries = append(mroot.Entries, newEntry)
@ -192,18 +189,10 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
newEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
newEntry = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@ -237,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := swarm.Manifest{}
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -250,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" {
// Replace the hash for leaf Manifest
newMRoot := swarm.Manifest{}
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
myEntry := swarm.ManifestEntry{
Path: entry.Path,
myEntry := api.ManifestEntry{
Hash: hash,
Path: entry.Path,
ContentType: ctype,
}
newMRoot.Entries = append(newMRoot.Entries, myEntry)
@ -276,18 +265,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
entryToRemove = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
entryToRemove = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@ -319,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := swarm.Manifest{}
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -331,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" {
// remove the entry in this Manifest
newMRoot := swarm.Manifest{}
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)

80
cmd/swarm/upload.go
View File

@ -18,13 +18,15 @@
package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
"os/user"
"path"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
@ -42,12 +44,10 @@ func upload(ctx *cli.Context) {
defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name)
fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
mimeType = ctx.GlobalString(SwarmUploadMimeType.Name)
client = swarm.NewClient(bzzapi)
file string
)
var client = swarm.NewClient(bzzapi)
var entry swarm.ManifestEntry
var file string
if len(args) != 1 {
if fromStdin {
tmp, err := ioutil.TempFile("", "swarm-stdin")
@ -66,41 +66,47 @@ func upload(ctx *cli.Context) {
utils.Fatalf("Need filename as the first and only argument")
}
} else {
file = args[0]
file = expandPath(args[0])
}
fi, err := os.Stat(expandPath(file))
if err != nil {
utils.Fatalf("Failed to stat file: %v", err)
if !wantManifest {
f, err := swarm.Open(file)
if err != nil {
utils.Fatalf("Error opening file: %s", err)
}
defer f.Close()
hash, err := client.UploadRaw(f, f.Size)
if err != nil {
utils.Fatalf("Upload failed: %s", err)
}
fmt.Println(hash)
return
}
if fi.IsDir() {
stat, err := os.Stat(file)
if err != nil {
utils.Fatalf("Error opening file: %s", err)
}
var hash string
if stat.IsDir() {
if !recursive {
utils.Fatalf("Argument is a directory and recursive upload is disabled")
}
if !wantManifest {
utils.Fatalf("Manifest is required for directory uploads")
hash, err = client.UploadDirectory(file, defaultPath, "")
} else {
if mimeType == "" {
mimeType = detectMimeType(file)
}
mhash, err := client.UploadDirectory(file, defaultPath)
f, err := swarm.Open(file)
if err != nil {
utils.Fatalf("Failed to upload directory: %v", err)
utils.Fatalf("Error opening file: %s", err)
}
fmt.Println(mhash)
return
defer f.Close()
f.ContentType = mimeType
hash, err = client.Upload(f, "")
}
entry, err = client.UploadFile(file, fi, mimeType)
if err != nil {
utils.Fatalf("Upload failed: %v", err)
}
mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
hash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
utils.Fatalf("Upload failed: %s", err)
}
fmt.Println(hash)
}
@ -128,3 +134,19 @@ func homeDir() string {
}
return ""
}
func detectMimeType(file string) string {
if ext := filepath.Ext(file); ext != "" {
return mime.TypeByExtension(ext)
}
f, err := os.Open(file)
if err != nil {
return ""
}
defer f.Close()
buf := make([]byte, 512)
if n, _ := f.Read(buf); n > 0 {
return http.DetectContentType(buf)
}
return ""
}

104
swarm/api/api.go
View File

@ -17,6 +17,7 @@
package api
import (
"errors"
"fmt"
"io"
"net/http"
@ -70,86 +71,50 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor
type ErrResolve error
// DNS Resolver
func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
log.Trace(fmt.Sprintf("Resolving : %v", hostPort))
if hashMatcher.MatchString(hostPort) || self.dns == nil {
log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort))
return storage.Key(common.Hex2Bytes(hostPort)), nil
func (self *Api) Resolve(uri *URI) (storage.Key, error) {
log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
if hashMatcher.MatchString(uri.Addr) {
log.Trace(fmt.Sprintf("addr is a hash: %q", uri.Addr))
return storage.Key(common.Hex2Bytes(uri.Addr)), nil
}
if !nameresolver {
return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort)
if uri.Immutable() {
return nil, errors.New("refusing to resolve immutable address")
}
contentHash, err := self.dns.Resolve(hostPort)
if self.dns == nil {
return nil, fmt.Errorf("unable to resolve addr %q, resolver not configured", uri.Addr)
}
hash, err := self.dns.Resolve(uri.Addr)
if err != nil {
err = ErrResolve(err)
log.Warn(fmt.Sprintf("DNS error : %v", err))
log.Warn(fmt.Sprintf("DNS error resolving addr %q: %s", uri.Addr, err))
return nil, ErrResolve(err)
}
log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash))
return contentHash[:], err
}
func Parse(uri string) (hostPort, path string) {
if uri == "" {
return
}
parts := slashes.Split(uri, 3)
var i int
if len(parts) == 0 {
return
}
// beginning with slash is now optional
for len(parts[i]) == 0 {
i++
}
hostPort = parts[i]
for i < len(parts)-1 {
i++
if len(path) > 0 {
path = path + "/" + parts[i]
} else {
path = parts[i]
}
}
log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path))
return
}
func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
hostPort, path = Parse(uri)
//resolving host and port
contentHash, err := self.Resolve(hostPort, nameresolver)
log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path))
return contentHash[:], hostPort, path, err
log.Trace(fmt.Sprintf("addr lookup: %v -> %v", uri.Addr, hash))
return hash[:], nil
}
// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (string, error) {
func (self *Api) Put(content, contentType string) (storage.Key, error) {
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
if err != nil {
return "", err
return nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
if err != nil {
return "", err
return nil, err
}
wg.Wait()
return key.String(), nil
return key, nil
}
// Get uses iterative manifest retrieval and prefix matching
// to resolve path to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
key, _, path, err := self.parseAndResolve(uri, nameresolver)
if err != nil {
return nil, "", 500, fmt.Errorf("can't resolve: %v", err)
}
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, key, quitC)
func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
trie, err := loadManifest(self.dpa, key, nil)
if err != nil {
log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
return
@ -173,32 +138,25 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
return
}
func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
root, _, path, err := self.parseAndResolve(uri, nameresolver)
if err != nil {
return "", fmt.Errorf("can't resolve: %v", err)
}
func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, root, quitC)
trie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
return
return nil, err
}
if contentHash != "" {
entry := &manifestTrieEntry{
entry := newManifestTrieEntry(&ManifestEntry{
Path: path,
Hash: contentHash,
ContentType: contentType,
}
}, nil)
entry.Hash = contentHash
trie.addEntry(entry, quitC)
} else {
trie.deleteEntry(path, quitC)
}
err = trie.recalcAndStore()
if err != nil {
return
if err := trie.recalcAndStore(); err != nil {
return nil, err
}
return trie.hash.String(), nil
return trie.hash, nil
}

10
swarm/api/api_test.go
View File

@ -23,6 +23,7 @@ import (
"os"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response {
}
// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
reader, mimeType, status, err := api.Get(bzzhash, true)
func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
key := storage.Key(common.Hex2Bytes(bzzhash))
reader, mimeType, status, err := api.Get(key, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
bzzhash, err := api.Put(content, exp.MimeType)
key, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
resp := testGet(t, api, bzzhash)
resp := testGet(t, api, key.String(), "")
checkResponse(t, resp, exp)
})
}

615
swarm/api/client/client.go
View File

@ -17,18 +17,23 @@
package client
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
)
var (
@ -36,18 +41,6 @@ var (
DefaultClient = NewClient(DefaultGateway)
)
// Manifest represents a swarm manifest.
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest.
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
@ -59,160 +52,207 @@ type Client struct {
Gateway string
}
func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
// UploadRaw uploads raw data to swarm and returns the resulting hash
func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
if err != nil {
return "", fmt.Errorf("failed to upload empty manifest")
return "", err
}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
return "", err
}
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
if err != nil {
return "", err
}
req.ContentLength = size
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// DownloadRaw downloads raw data from swarm
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
uri := c.Gateway + "/bzzr:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return res.Body, nil
}
// File represents a file in a swarm manifest and is used for uploading and
// downloading content to and from swarm
type File struct {
io.ReadCloser
api.ManifestEntry
}
// Open opens a local file which can then be passed to client.Upload to upload
// it to swarm
func Open(path string) (*File, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
stat, err := f.Stat()
if err != nil {
f.Close()
return nil, err
}
return &File{
ReadCloser: f,
ManifestEntry: api.ManifestEntry{
ContentType: mime.TypeByExtension(filepath.Ext(path)),
Mode: int64(stat.Mode()),
Size: stat.Size(),
ModTime: stat.ModTime(),
},
}, nil
}
// Upload uploads a file to swarm and either adds it to an existing manifest
// (if the manifest argument is non-empty) or creates a new manifest containing
// the file, returning the resulting manifest hash (the file will then be
// available at bzz:/<hash>/<path>)
func (c *Client) Upload(file *File, manifest string) (string, error) {
if file.Size <= 0 {
return "", errors.New("file size must be greater than zero")
}
return c.TarUpload(manifest, &FileUploader{file})
}
// Download downloads a file with the given path from the swarm manifest with
// the given hash (i.e. it gets bzz:/<hash>/<path>)
func (c *Client) Download(hash, path string) (*File, error) {
uri := c.Gateway + "/bzz:/" + hash + "/" + path
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return &File{
ReadCloser: res.Body,
ManifestEntry: api.ManifestEntry{
ContentType: res.Header.Get("Content-Type"),
Size: res.ContentLength,
},
}, nil
}
// UploadDirectory uploads a directory tree to swarm and either adds the files
// to an existing manifest (if the manifest argument is non-empty) or creates a
// new manifest, returning the resulting manifest hash (files from the
// directory will then be available at bzz:/<hash>/path/to/file), with
// the file specified in defaultPath being uploaded to the root of the manifest
// (i.e. bzz:/<hash>/)
func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
stat, err := os.Stat(dir)
if err != nil {
return "", err
} else if !stat.IsDir() {
return "", fmt.Errorf("not a directory: %s", dir)
}
return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
}
// DownloadDirectory downloads the files contained in a swarm manifest under
// the given path into a local directory (existing files will be overwritten)
func (c *Client) DownloadDirectory(hash, path, destDir string) error {
stat, err := os.Stat(destDir)
if err != nil {
return err
} else if !stat.IsDir() {
return fmt.Errorf("not a directory: %s", destDir)
}
uri := c.Gateway + "/bzz:/" + hash + "/" + path
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
req.Header.Set("Accept", "application/x-tar")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
tr := tar.NewReader(res.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return err
}
if !strings.HasPrefix(path, dir) {
return fmt.Errorf("path %s outside directory %s", path, dir)
}
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
return err
})
return mhash, err
}
func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
var mimetype string
hash, err := c.uploadFileContent(file, fi)
if mimetype_hint != "" {
mimetype = mimetype_hint
log.Info("Mime type set by override", "mime", mimetype)
} else {
ext := filepath.Ext(file)
log.Info("Ext", "ext", ext, "file", file)
if ext != "" {
mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
} else {
f, err := os.Open(file)
if err == nil {
first512 := make([]byte, 512)
fread, _ := f.ReadAt(first512, 0)
if fread > 0 {
mimetype = http.DetectContentType(first512[:fread])
log.Info("Mime type set by autodetection", "mime", mimetype)
}
}
f.Close()
// ignore the default path file
if hdr.Name == "" {
continue
}
dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
return err
}
var mode os.FileMode = 0644
if hdr.Mode > 0 {
mode = os.FileMode(hdr.Mode)
}
dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
n, err := io.Copy(dst, tr)
dst.Close()
if err != nil {
return err
} else if n != hdr.Size {
return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
}
}
m := ManifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
fd, err := os.Open(file)
// UploadManifest uploads the given manifest to swarm
func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
data, err := json.Marshal(m)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
return c.postRaw("application/octet-stream", fi.Size(), fd)
return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
}
func (c *Client) UploadManifest(m Manifest) (string, error) {
jsm, err := json.Marshal(m)
// DownloadManifest downloads a swarm manifest
func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
res, err := c.DownloadRaw(hash)
if err != nil {
panic(err)
return nil, err
}
log.Info("Uploading swarm manifest")
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
defer res.Close()
var manifest api.Manifest
if err := json.NewDecoder(res).Decode(&manifest); err != nil {
return nil, err
}
return &manifest, nil
}
func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
fd, err := os.Open(fpath)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
if err != nil {
return "", err
}
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
if err != nil {
return "", err
}
req.Header.Set("content-type", mimetype)
req.ContentLength = size
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
mroot := Manifest{}
req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
if err != nil {
return mroot, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return mroot, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return mroot, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(content, &mroot)
if err != nil {
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
}
return mroot, err
}
// ManifestFileList downloads the manifest with the given hash and generates a
// list of files and directory prefixes which have the specified prefix.
// List list files in a swarm manifest which have the given prefix, grouping
// common prefixes using "/" as a delimiter.
//
// For example, if the manifest represents the following directory structure:
//
@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
manifest, err := c.DownloadManifest(hash)
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
if err != nil {
return nil, err
}
// handleFile handles a manifest entry which is a direct reference to a
// file (i.e. it is not a swarm manifest)
handleFile := func(entry ManifestEntry) {
// ignore the file if it doesn't have the specified prefix
if !strings.HasPrefix(entry.Path, prefix) {
return
}
// if the path after the prefix contains a directory separator,
// add a directory prefix to the entries, otherwise add the
// file
suffix := strings.TrimPrefix(entry.Path, prefix)
if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
entries = append(entries, ManifestEntry{
Path: prefix + suffix[:sepIndex+1],
ContentType: "DIR",
})
} else {
if entry.Path == "" {
entry.Path = "/"
}
entries = append(entries, entry)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
// handleManifest handles a manifest entry which is a reference to
// another swarm manifest.
handleManifest := func(entry ManifestEntry) error {
// if the manifest's path is a prefix of the specified prefix
// then just recurse into the manifest by stripping its path
// from the prefix
if strings.HasPrefix(prefix, entry.Path) {
subPrefix := strings.TrimPrefix(prefix, entry.Path)
subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
if err != nil {
return err
}
// prefix the manifest's path to the sub entries and
// add them to the returned entries
for i, subEntry := range subEntries {
subEntry.Path = entry.Path + subEntry.Path
subEntries[i] = subEntry
}
entries = append(entries, subEntries...)
return nil
}
// if the manifest's path has the specified prefix, then if the
// path after the prefix contains a directory separator, add a
// directory prefix to the entries, otherwise recurse into the
// manifest
if strings.HasPrefix(entry.Path, prefix) {
suffix := strings.TrimPrefix(entry.Path, prefix)
sepIndex := strings.Index(suffix, "/")
if sepIndex > -1 {
entries = append(entries, ManifestEntry{
Path: prefix + suffix[:sepIndex+1],
ContentType: "DIR",
})
return nil
}
subEntries, err := c.ManifestFileList(entry.Hash, "")
if err != nil {
return err
}
// prefix the manifest's path to the sub entries and
// add them to the returned entries
for i, subEntry := range subEntries {
subEntry.Path = entry.Path + subEntry.Path
subEntries[i] = subEntry
}
entries = append(entries, subEntries...)
return nil
}
return nil
var list api.ManifestList
if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
return nil, err
}
for _, entry := range manifest.Entries {
if entry.ContentType == "application/bzz-manifest+json" {
if err := handleManifest(entry); err != nil {
return nil, err
}
} else {
handleFile(entry)
}
}
return
return &list, nil
}
// Uploader uploads files to swarm using a provided UploadFn
type Uploader interface {
Upload(UploadFn) error
}
type UploaderFunc func(UploadFn) error
func (u UploaderFunc) Upload(upload UploadFn) error {
return u(upload)
}
// DirectoryUploader uploads all files in a directory, optionally uploading
// a file to the default path
type DirectoryUploader struct {
Dir string
DefaultPath string
}
// Upload performs the upload of the directory and default path
func (d *DirectoryUploader) Upload(upload UploadFn) error {
if d.DefaultPath != "" {
file, err := Open(d.DefaultPath)
if err != nil {
return err
}
if err := upload(file); err != nil {
return err
}
}
return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if f.IsDir() {
return nil
}
file, err := Open(path)
if err != nil {
return err
}
relPath, err := filepath.Rel(d.Dir, path)
if err != nil {
return err
}
file.Path = filepath.ToSlash(relPath)
return upload(file)
})
}
// FileUploader uploads a single file
type FileUploader struct {
File *File
}
// Upload performs the upload of the file
func (f *FileUploader) Upload(upload UploadFn) error {
return upload(f.File)
}
// UploadFn is the type of function passed to an Uploader to perform the upload
// of a single file (for example, a directory uploader would call a provided
// UploadFn for each file in the directory tree)
type UploadFn func(file *File) error
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
// returning the resulting manifest hash
func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/x-tar")
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
tw := tar.NewWriter(reqW)
// define an UploadFn which adds files to the tar stream
uploadFn := func(file *File) error {
hdr := &tar.Header{
Name: file.Path,
Mode: file.Mode,
Size: file.Size,
ModTime: file.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": file.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the tar stream
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = tw.Close()
}
reqW.CloseWithError(err)
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// MultipartUpload uses the given Uploader to upload files to swarm as a
// multipart form, returning the resulting manifest hash
func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
if err != nil {
return "", err
}
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
mw := multipart.NewWriter(reqW)
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
// define an UploadFn which adds files to the multipart form
uploadFn := func(file *File) error {
hdr := make(textproto.MIMEHeader)
hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
hdr.Set("Content-Type", file.ContentType)
hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
w, err := mw.CreatePart(hdr)
if err != nil {
return err
}
_, err = io.Copy(w, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the multipart form
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = mw.Close()
}
reqW.CloseWithError(err)
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}

262
swarm/api/client/client_test.go
View File

@ -17,6 +17,7 @@
package client
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
@ -24,52 +25,221 @@ import (
"sort"
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestClientManifestFileList(t *testing.T) {
// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
func TestClientUploadDownloadRaw(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
client := NewClient(srv.URL)
// upload some raw data
data := []byte("foo123")
hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
if err != nil {
t.Fatal(err)
}
// check we can download the same data
res, err := client.DownloadRaw(hash)
if err != nil {
t.Fatal(err)
}
defer res.Close()
gotData, err := ioutil.ReadAll(res)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
}
}
// TestClientUploadDownloadFiles test uploading and downloading files to swarm
// manifests
func TestClientUploadDownloadFiles(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
client := NewClient(srv.URL)
upload := func(manifest, path string, data []byte) string {
file := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: path,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
hash, err := client.Upload(file, manifest)
if err != nil {
t.Fatal(err)
}
return hash
}
checkDownload := func(manifest, path string, expected []byte) {
file, err := client.Download(manifest, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
if file.Size != int64(len(expected)) {
t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
}
if file.ContentType != file.ContentType {
t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
}
data, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
}
}
// upload a file to the root of a manifest
rootData := []byte("some-data")
rootHash := upload("", "", rootData)
// check we can download the root file
checkDownload(rootHash, "", rootData)
// upload another file to the same manifest
otherData := []byte("some-other-data")
newHash := upload(rootHash, "some/other/path", otherData)
// check we can download both files from the new manifest
checkDownload(newHash, "", rootData)
checkDownload(newHash, "some/other/path", otherData)
// replace the root file with different data
newHash = upload(newHash, "", otherData)
// check both files have the other data
checkDownload(newHash, "", otherData)
checkDownload(newHash, "some/other/path", otherData)
}
var testDirFiles = []string{
"file1.txt",
"file2.txt",
"dir1/file3.txt",
"dir1/file4.txt",
"dir2/file5.txt",
"dir2/dir3/file6.txt",
"dir2/dir4/file7.txt",
"dir2/dir4/file8.txt",
}
func newTestDirectory(t *testing.T) string {
dir, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
files := []string{
"file1.txt",
"file2.txt",
"dir1/file3.txt",
"dir1/file4.txt",
"dir2/file5.txt",
"dir2/dir3/file6.txt",
"dir2/dir4/file7.txt",
"dir2/dir4/file8.txt",
}
for _, file := range files {
for _, file := range testDirFiles {
path := filepath.Join(dir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
os.RemoveAll(dir)
t.Fatalf("error creating dir for %s: %s", path, err)
}
if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
os.RemoveAll(dir)
t.Fatalf("error writing file %s: %s", path, err)
}
}
client := NewClient(srv.URL)
return dir
}
hash, err := client.UploadDirectory(dir, "")
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
// upload the directory
client := NewClient(srv.URL)
defaultPath := filepath.Join(dir, testDirFiles[0])
hash, err := client.UploadDirectory(dir, defaultPath, "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
// check we can download the individual files
checkDownloadFile := func(path string, expected []byte) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
data, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected data to be %q, got %q", expected, data)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file, []byte(file))
}
// check we can download the default path
checkDownloadFile("", []byte(testDirFiles[0]))
// check we can download the directory
tmp, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
if err := client.DownloadDirectory(hash, "", tmp); err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
data, err := ioutil.ReadFile(filepath.Join(tmp, file))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, []byte(file)) {
t.Fatalf("expected data to be %q, got %q", file, data)
}
}
}
// TestClientFileList tests listing files in a swarm manifest
func TestClientFileList(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
client := NewClient(srv.URL)
hash, err := client.UploadDirectory(dir, "", "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
ls := func(prefix string) []string {
entries, err := client.ManifestFileList(hash, prefix)
list, err := client.List(hash, prefix)
if err != nil {
t.Fatal(err)
}
paths := make([]string, len(entries))
for i, entry := range entries {
paths[i] = entry.Path
paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
for _, prefix := range list.CommonPrefixes {
paths = append(paths, prefix)
}
for _, entry := range list.Entries {
paths = append(paths, entry.Path)
}
sort.Strings(paths)
return paths
@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
for prefix, expected := range tests {
actual := ls(prefix)
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
}
}
}
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
data := []byte("some-data")
uploader := UploaderFunc(func(upload UploadFn) error {
for _, name := range testDirFiles {
file := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: name,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
if err := upload(file); err != nil {
return err
}
}
return nil
})
// upload the files as a multipart upload
client := NewClient(srv.URL)
hash, err := client.MultipartUpload("", uploader)
if err != nil {
t.Fatal(err)
}
// check we can download the individual files
checkDownloadFile := func(path string) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
gotData, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected data to be %q, got %q", data, gotData)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file)
}
}

27
swarm/api/filesystem.go
View File

@ -22,6 +22,7 @@ import (
"io"
"net/http"
"os"
"path"
"path/filepath"
"sync"
@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem {
// Upload replicates a local directory as a manifest file and uploads it
// using dpa store
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Upload(lpath, index string) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
@ -72,9 +75,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if path[:start] != localpath {
return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
}
entry := &manifestTrieEntry{
Path: filepath.ToSlash(path),
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
list = append(list, entry)
}
return err
@ -91,9 +92,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if localpath[:start] != dir {
return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
}
entry := &manifestTrieEntry{
Path: filepath.ToSlash(localpath),
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
list = append(list, entry)
}
@ -153,11 +152,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
}
entry.Path = RegularSlashes(entry.Path[start:])
if entry.Path == index {
ientry := &manifestTrieEntry{
Path: "",
Hash: entry.Hash,
ientry := newManifestTrieEntry(&ManifestEntry{
ContentType: entry.ContentType,
}
}, nil)
ientry.Hash = entry.Hash
trie.addEntry(ientry, quitC)
}
trie.addEntry(entry, quitC)
@ -174,6 +172,8 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
// Download replicates the manifest path structure on the local filesystem
// under localpath
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
@ -185,10 +185,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
//resolving host and port
key, _, path, err := self.api.parseAndResolve(bzzpath, true)
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return err
}
key, err := self.api.Resolve(uri)
if err != nil {
return err
}
path := uri.Path
if len(path) > 0 {
path += "/"

32
swarm/api/filesystem_test.go
View File

@ -23,6 +23,9 @@ import (
"path/filepath"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash+"/index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash+"/index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
_, _, _, err = api.Get(bzzhash, true)
key := storage.Key(common.Hex2Bytes(bzzhash))
_, _, _, err = api.Get(key, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) {
return
}
bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
key := storage.Key(common.Hex2Bytes(bzzhash))
key, err = api.Modify(key, "index.html", "", "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash = key.String()
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash+"/index2.html")
resp := testGet(t, api, bzzhash, "index2.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
resp = testGet(t, api, bzzhash+"/img/logo.png")
resp = testGet(t, api, bzzhash, "img/logo.png")
exp = expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash+"/index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
_, _, _, err = api.Get(bzzhash, true)
_, _, _, err = api.Get(key, "")
if err == nil {
t.Errorf("expected error: %v", err)
}
@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash)
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash+"/index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash)
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})

11
swarm/api/http/roundtripper_test.go
View File

@ -18,14 +18,14 @@ package http
import (
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
const port = "3222"
func TestRoundTripper(t *testing.T) {
serveMux := http.NewServeMux()
serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) {
http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
}
})
go http.ListenAndServe(":"+port, serveMux)
rt := &RoundTripper{Port: port}
srv := httptest.NewServer(serveMux)
defer srv.Close()
host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
rt := &RoundTripper{Host: host, Port: port}
trans := &http.Transport{}
trans.RegisterProtocol("bzz", rt)
client := &http.Client{Transport: trans}

775
swarm/api/http/server.go
View File

@ -20,13 +20,19 @@ A simple http server interface to Swarm
package http
import (
"bytes"
"archive/tar"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"regexp"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
@ -36,26 +42,6 @@ import (
"github.com/rs/cors"
)
const (
rawType = "application/octet-stream"
)
var (
// accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+")
trailingSlashes = regexp.MustCompile("/+$")
rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$")
// forever = func() time.Time { return time.Unix(0, 0) }
forever = time.Now
)
type sequentialReader struct {
reader io.Reader
pos int64
ahead map[int64](chan bool)
lock sync.Mutex
}
// ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings.
type ServerConfig struct {
@ -94,242 +80,569 @@ type Server struct {
api *api.Api
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestURL := r.URL
// This is wrong
// if requestURL.Host == "" {
// var err error
// requestURL, err = url.Parse(r.Referer() + requestURL.String())
// if err != nil {
// http.Error(w, err.Error(), http.StatusBadRequest)
// return
// }
// }
log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")))
uri := requestURL.Path
var raw, nameresolver bool
var proto string
// Request wraps http.Request and also includes the parsed bzz URI
type Request struct {
http.Request
// HTTP-based URL protocol handler
log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri))
uri *api.URI
}
path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
proto = p
return ""
})
// protocol identification (ugly)
if proto == "" {
log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri))
http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest)
// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
s.BadRequest(w, r, "raw POST request cannot contain a path")
return
}
if len(proto) > 4 {
raw = proto[1:5] == "bzzr"
nameresolver = proto[1:5] != "bzzi"
if r.Header.Get("Content-Length") == "" {
s.BadRequest(w, r, "missing Content-Length header in request")
return
}
log.Debug("", "msg", log.Lazy{Fn: func() string {
return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path)
}})
key, err := s.api.Store(r.Body, r.ContentLength, nil)
if err != nil {
s.Error(w, r, err)
return
}
s.logDebug("content for %s stored", key.Log())
switch {
case r.Method == "POST" || r.Method == "PUT":
if r.Header.Get("content-length") == "" {
http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, key)
}
// HandlePostFiles handles a POST request (or deprecated PUT request) to
// bzz:/<hash>/<path> which contains either a single file or multiple files
// (either a tar archive or multipart form), adds those files either to an
// existing manifest or to a new manifest under <path> and returns the
// resulting manifest hash as a text/plain response
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
s.BadRequest(w, r, err.Error())
return
}
var key storage.Key
if r.uri.Addr != "" {
key, err = s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
if err == nil {
log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
} else {
key, err = s.api.NewManifest()
if err != nil {
s.Error(w, r, err)
return
}
if r.Method == "POST" {
if raw {
w.Header().Set("Content-Type", "text/plain")
http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key))))
} else {
http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
return
}
} else {
// PUT
if raw {
http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest)
return
} else {
path = api.RegularSlashes(path)
mime := r.Header.Get("Content-Type")
// TODO proper root hash separation
log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain")
http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
} else {
http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest)
return
}
}
}
case r.Method == "DELETE":
if raw {
http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest)
return
} else {
path = api.RegularSlashes(path)
log.Debug(fmt.Sprintf("Delete '%s'.", path))
newKey, err := s.api.Modify(path, "", "", nameresolver)
if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain")
http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
} else {
http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest)
return
}
}
case r.Method == "GET" || r.Method == "HEAD":
path = trailingSlashes.ReplaceAllString(path, "")
if path == "" {
http.Error(w, "Empty path not allowed", http.StatusBadRequest)
return
}
if raw {
var reader storage.LazySectionReader
parsedurl, _ := api.Parse(path)
}
if parsedurl == path {
key, err := s.api.Resolve(parsedurl, nameresolver)
if err != nil {
log.Error(fmt.Sprintf("%v", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
reader = s.api.Retrieve(key)
} else {
var status int
readertmp, _, status, err := s.api.Get(path, nameresolver)
if err != nil {
http.Error(w, err.Error(), status)
return
}
reader = readertmp
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
switch contentType {
// retrieving content
case "application/x-tar":
return s.handleTarUpload(r, mw)
quitC := make(chan bool)
size, err := reader.Size(quitC)
case "multipart/form-data":
return s.handleMultipartUpload(r, params["boundary"], mw)
default:
return s.handleDirectUpload(r, mw)
}
})
if err != nil {
s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
tr := tar.NewReader(req.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading tar stream: %s", err)
}
// only store regular files
if !hdr.FileInfo().Mode().IsRegular() {
continue
}
// add the entry under the path from the request
path := path.Join(req.uri.Path, hdr.Name)
entry := &api.ManifestEntry{
Path: path,
ContentType: hdr.Xattrs["user.swarm.content-type"],
Mode: hdr.Mode,
Size: hdr.Size,
ModTime: hdr.ModTime,
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(tr, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
mr := multipart.NewReader(req.Body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading multipart form: %s", err)
}
var size int64
var reader io.Reader = part
if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
//An error on call to Size means we don't have the root chunk
http.Error(w, err.Error(), http.StatusNotFound)
return
return fmt.Errorf("error parsing multipart content length: %s", err)
}
log.Debug(fmt.Sprintf("Reading %d bytes.", size))
// setting mime type
qv := requestURL.Query()
mimeType := qv.Get("content_type")
if mimeType == "" {
mimeType = rawType
}
w.Header().Set("Content-Type", mimeType)
http.ServeContent(w, r, uri, forever(), reader)
log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType))
// retrieve path via manifest
reader = part
} else {
log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri))
// add trailing slash, if missing
if rootDocumentUri.MatchString(uri) {
http.Redirect(w, r, path+"/", http.StatusFound)
return
}
reader, mimeType, status, err := s.api.Get(path, nameresolver)
// copy the part to a tmp file to get its size
tmp, err := ioutil.TempFile("", "swarm-multipart")
if err != nil {
if _, ok := err.(api.ErrResolve); ok {
log.Debug(fmt.Sprintf("%v", err))
status = http.StatusBadRequest
} else {
log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err))
status = http.StatusNotFound
}
http.Error(w, err.Error(), status)
return
return err
}
// set mime type and status headers
w.Header().Set("Content-Type", mimeType)
if status > 0 {
w.WriteHeader(status)
} else {
status = 200
}
quitC := make(chan bool)
size, err := reader.Size(quitC)
defer os.Remove(tmp.Name())
defer tmp.Close()
size, err = io.Copy(tmp, part)
if err != nil {
log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
//An error on call to Size means we don't have the root chunk
http.Error(w, err.Error(), http.StatusNotFound)
return
return fmt.Errorf("error copying multipart content: %s", err)
}
log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status))
http.ServeContent(w, r, path, forever(), reader)
if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
reader = tmp
}
// add the entry under the path from the request
name := part.FileName()
if name == "" {
name = part.FormName()
}
path := path.Join(req.uri.Path, name)
entry := &api.ManifestEntry{
Path: path,
ContentType: part.Header.Get("Content-Type"),
Size: size,
ModTime: time.Now(),
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
Path: req.uri.Path,
ContentType: req.Header.Get("Content-Type"),
Mode: 0644,
Size: req.ContentLength,
ModTime: time.Now(),
})
if err != nil {
return err
}
s.logDebug("content for %s stored", key.Log())
return nil
}
// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
// <path> from <manifest> and returns the resulting manifest hash as a
// text/plain response
func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
return mw.RemoveEntry(r.uri.Path)
})
if err != nil {
s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
// HandleGetRaw handles a GET request to bzzr://<key> and responds with
// the raw content stored at the given storage key
func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
if r.uri.Path != "" {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
return
}
var entry *api.ManifestEntry
walker.Walk(func(e *api.ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == r.uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
}
// ignore non-manifest files
if e.ContentType != api.ManifestType {
return nil
}
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(r.uri.Path, e.Path) {
return nil
}
return api.SkipManifest
})
if entry == nil {
http.NotFound(w, &r.Request)
return
}
key = storage.Key(common.Hex2Bytes(entry.Hash))
}
// check the root chunk exists by retrieving the file's size
reader := s.api.Retrieve(key)
if _, err := reader.Size(nil); err != nil {
s.logDebug("key not found %s: %s", key, err)
http.NotFound(w, &r.Request)
return
}
// allow the request to overwrite the content type using a query
// parameter
contentType := "application/octet-stream"
if typ := r.URL.Query().Get("content_type"); typ != "" {
contentType = typ
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
}
// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
// header of "application/x-tar" and returns a tar stream of all files
// contained in the manifest
func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
s.BadRequest(w, r, "files request cannot contain a path")
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
s.Error(w, r, err)
return
}
tw := tar.NewWriter(w)
defer tw.Close()
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
err = walker.Walk(func(entry *api.ManifestEntry) error {
// ignore manifests (walk will recurse into them)
if entry.ContentType == api.ManifestType {
return nil
}
// retrieve the entry's key and size
reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
size, err := reader.Size(nil)
if err != nil {
return err
}
// write a tar header for the entry
hdr := &tar.Header{
Name: entry.Path,
Mode: entry.Mode,
Size: size,
ModTime: entry.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": entry.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
// copy the file into the tar stream
n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
if err != nil {
return err
} else if n != size {
return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
}
return nil
})
if err != nil {
s.logError("error generating tar stream: %s", err)
}
}
// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
// the "list" query parameter set to "true" and returns a list of all files
// contained in <manifest> under <path> grouped into common prefixes using
// "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
s.Error(w, r, err)
return
}
var list api.ManifestList
prefix := r.uri.Path
err = walker.Walk(func(entry *api.ManifestEntry) error {
// handle non-manifest files
if entry.ContentType != api.ManifestType {
// ignore the file if it doesn't have the specified prefix
if !strings.HasPrefix(entry.Path, prefix) {
return nil
}
// if the path after the prefix contains a slash, add a
// common prefix to the list, otherwise add the entry
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return nil
}
if entry.Path == "" {
entry.Path = "/"
}
list.Entries = append(list.Entries, entry)
return nil
}
// if the manifest's path is a prefix of the specified prefix
// then just recurse into the manifest by returning nil and
// continuing the walk
if strings.HasPrefix(prefix, entry.Path) {
return nil
}
// if the manifest's path has the specified prefix, then if the
// path after the prefix contains a slash, add a common prefix
// to the list and skip the manifest, otherwise recurse into
// the manifest by returning nil and continuing the walk
if strings.HasPrefix(entry.Path, prefix) {
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return api.SkipManifest
}
return nil
}
// the manifest neither has the prefix or needs recursing in to
// so just skip it
return api.SkipManifest
})
if err != nil {
s.Error(w, r, err)
return
}
// if the client wants HTML (e.g. a browser) then render the list as a
// HTML index with relative URLs
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
URI: r.uri,
List: &list,
})
if err != nil {
s.logError("error rendering list HTML: %s", err)
}
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(&list)
}
// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
// with the content of the file at <path> from the given <manifest>
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
reader, contentType, _, err := s.api.Get(key, r.uri.Path)
if err != nil {
s.Error(w, r, err)
return
}
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(nil); err != nil {
s.logDebug("file not found %s: %s", r.uri, err)
http.NotFound(w, &r.Request)
return
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
if err != nil {
s.logError("Invalid URI %q: %s", r.URL.Path, err)
http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest)
return
}
s.logDebug("%s request received for %s", r.Method, uri)
req := &Request{Request: *r, uri: uri}
switch r.Method {
case "POST":
if uri.Raw() {
s.HandlePostRaw(w, req)
} else {
s.HandlePostFiles(w, req)
}
case "PUT":
// DEPRECATED:
// clients should send a POST request (the request creates a
// new manifest leaving the existing one intact, so it isn't
// strictly a traditional PUT request which replaces content
// at a URI, and POST is more ubiquitous)
if uri.Raw() {
http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
} else {
s.HandlePostFiles(w, req)
}
case "DELETE":
if uri.Raw() {
http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
return
}
s.HandleDelete(w, req)
case "GET":
if uri.Raw() {
s.HandleGetRaw(w, req)
return
}
if r.Header.Get("Accept") == "application/x-tar" {
s.HandleGetFiles(w, req)
return
}
if r.URL.Query().Get("list") == "true" {
s.HandleGetList(w, req)
return
}
s.HandleGetFile(w, req)
default:
http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
}
}
func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) {
self.lock.Lock()
// assert self.pos <= off
if self.pos > off {
log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off))
panic("Non-sequential read attempt")
func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
mw, err := s.api.NewManifestWriter(key, nil)
if err != nil {
return nil, err
}
if self.pos != off {
log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off))
wait := make(chan bool)
self.ahead[off] = wait
self.lock.Unlock()
if <-wait {
// failed read behind
n = 0
err = io.ErrUnexpectedEOF
return
}
self.lock.Lock()
if err := update(mw); err != nil {
return nil, err
}
localPos := 0
for localPos < len(target) {
n, err = self.reader.Read(target[localPos:])
localPos += n
log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err))
if err != nil {
log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err))
for i := range self.ahead {
self.ahead[i] <- true
delete(self.ahead, i)
}
self.lock.Unlock()
return localPos, err
}
self.pos += int64(n)
key, err = mw.Store()
if err != nil {
return nil, err
}
wait := self.ahead[self.pos]
if wait != nil {
log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos))
delete(self.ahead, self.pos)
close(wait)
}
self.lock.Unlock()
return localPos, err
s.logDebug("generated manifest %s", key)
return key, nil
}
func (s *Server) logDebug(format string, v ...interface{}) {
log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) logError(format string, v ...interface{}) {
log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason)
http.Error(w, reason, http.StatusBadRequest)
}
func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
s.logError("error serving %s %s: %s", r.Method, r.uri, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
}

4
swarm/api/http/server_test.go
View File

@ -40,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) {
testrequests := make(map[string]int)
testrequests["/"] = 0
testrequests["/a"] = 1
testrequests["/a/b"] = 2
testrequests["/a/"] = 1
testrequests["/a/b/"] = 2
testrequests["/x"] = 0
testrequests[""] = 0

71
swarm/api/http/templates.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"html/template"
"path"
"github.com/ethereum/go-ethereum/swarm/api"
)
type htmlListData struct {
URI *api.URI
List *api.ManifestList
}
var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(`
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Swarm index of {{ .URI }}</title>
</head>
<body>
<h1>Swarm index of {{ .URI }}</h1>
<hr>
<table>
<thead>
<tr>
<th>Path</th>
<th>Type</th>
<th>Size</th>
</tr>
</thead>
<tbody>
{{ range .List.CommonPrefixes }}
<tr>
<td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
<td>DIR</td>
<td>-</td>
</tr>
{{ end }}
{{ range .List.Entries }}
<tr>
<td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td>
<td>{{ .ContentType }}</td>
<td>{{ .Size }}</td>
</tr>
{{ end }}
</table>
<hr>
</body>
`[1:]))

170
swarm/api/manifest.go
View File

@ -19,8 +19,11 @@ package api
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
@ -28,25 +31,152 @@ import (
)
const (
manifestType = "application/bzz-manifest+json"
ManifestType = "application/bzz-manifest+json"
)
// Manifest represents a swarm manifest
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
type ManifestList struct {
CommonPrefixes []string `json:"common_prefixes,omitempty"`
Entries []*ManifestEntry `json:"entries,omitempty"`
}
// NewManifest creates and stores a new, empty manifest
func (a *Api) NewManifest() (storage.Key, error) {
var manifest Manifest
data, err := json.Marshal(&manifest)
if err != nil {
return nil, err
}
return a.Store(bytes.NewReader(data), int64(len(data)), nil)
}
// ManifestWriter is used to add and remove entries from an underlying manifest
type ManifestWriter struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWriter{a, trie, quitC}, nil
}
// AddEntry stores the given data and adds the resulting key to the manifest
func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
key, err := m.api.Store(data, e.Size, nil)
if err != nil {
return nil, err
}
entry := newManifestTrieEntry(e, nil)
entry.Hash = key.String()
m.trie.addEntry(entry, m.quitC)
return key, nil
}
// RemoveEntry removes the given path from the manifest
func (m *ManifestWriter) RemoveEntry(path string) error {
m.trie.deleteEntry(path, m.quitC)
return nil
}
// Store stores the manifest, returning the resulting storage key
func (m *ManifestWriter) Store() (storage.Key, error) {
return m.trie.hash, m.trie.recalcAndStore()
}
// ManifestWalker is used to recursively walk the entries in the manifest and
// all of its submanifests
type ManifestWalker struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWalker{a, trie, quitC}, nil
}
// SkipManifest is used as a return value from WalkFn to indicate that the
// manifest should be skipped
var SkipManifest = errors.New("skip this manifest")
// WalkFn is the type of function called for each entry visited by a recursive
// manifest walk
type WalkFn func(entry *ManifestEntry) error
// Walk recursively walks the manifest calling walkFn for each entry in the
// manifest, including submanifests
func (m *ManifestWalker) Walk(walkFn WalkFn) error {
return m.walk(m.trie, "", walkFn)
}
func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
for _, entry := range trie.entries {
if entry == nil {
continue
}
entry.Path = prefix + entry.Path
err := walkFn(&entry.ManifestEntry)
if err != nil {
if entry.ContentType == ManifestType && err == SkipManifest {
continue
}
return err
}
if entry.ContentType != ManifestType {
continue
}
if err := trie.loadSubTrie(entry, nil); err != nil {
return err
}
if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
return err
}
}
return nil
}
type manifestTrie struct {
dpa *storage.DPA
entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
hash storage.Key // if hash != nil, it is stored
}
type manifestJSON struct {
Entries []*manifestTrieEntry `json:"entries"`
func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
return &manifestTrieEntry{
ManifestEntry: *entry,
subtrie: subtrie,
}
}
type manifestTrieEntry struct {
Path string `json:"path"`
Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
ContentType string `json:"contentType"`
Status int `json:"status"`
subtrie *manifestTrie
ManifestEntry
subtrie *manifestTrie
}
func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
}
log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
man := manifestJSON{}
var man struct {
Entries []*manifestTrieEntry `json:"entries"`
}
err = json.Unmarshal(manifestData, &man)
if err != nil {
err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
cpl++
}
if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry, quitC) != nil {
return
}
@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
self.entries[b] = &manifestTrieEntry{
self.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
Hash: "",
ContentType: manifestType,
subtrie: subtrie,
}
ContentType: ManifestType,
}, subtrie)
}
func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
}
epl := len(entry.Path)
if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry, quitC) != nil {
return
}
@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error {
var buffer bytes.Buffer
buffer.WriteString(`{"entries":[`)
list := &manifestJSON{}
list := &Manifest{}
for _, entry := range self.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error {
}
entry.Hash = entry.subtrie.hash.String()
}
list.Entries = append(list.Entries, entry)
list.Entries = append(list.Entries, entry.ManifestEntry)
}
}
@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
entry := self.entries[i]
if entry != nil {
epl := len(entry.Path)
if entry.ContentType == manifestType {
if entry.ContentType == ManifestType {
l := plen
if epl < l {
l = epl
@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
if (len(path) >= epl) && (path[:epl] == entry.Path) {
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
if entry.ContentType == manifestType {
if entry.ContentType == ManifestType {
err := self.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0

40
swarm/api/storage.go
View File

@ -16,6 +16,8 @@
package api
import "path"
type Response struct {
MimeType string
Status int
@ -25,6 +27,8 @@ type Response struct {
}
// implements a service
//
// DEPRECATED: Use the HTTP API instead
type Storage struct {
api *Api
}
@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage {
// Put uploads the content to the swarm with a simple manifest speficying
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
return self.api.Put(content, contentType)
key, err := self.api.Put(content, contentType)
if err != nil {
return "", err
}
return key.String(), err
}
// Get retrieves the content from bzzpath and reads the response in full
@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) {
// NOTE: if error is non-nil, sResponse may still have partial content
// the actual size of which is given in len(resp.Content), while the expected
// size is resp.Size
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Get(bzzpath string) (*Response, error) {
reader, mimeType, status, err := self.api.Get(bzzpath, true)
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return nil, err
}
key, err := self.api.Resolve(uri)
if err != nil {
return nil, err
}
reader, mimeType, status, err := self.api.Get(key, uri.Path)
if err != nil {
return nil, err
}
@ -65,6 +85,20 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
// and merge on to it. creating an entry w conentType (mime)
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
uri, err := Parse("bzz:/" + rootHash)
if err != nil {
return "", err
}
key, err := self.api.Resolve(uri)
if err != nil {
return "", err
}
key, err = self.api.Modify(key, path, contentHash, contentType)
if err != nil {
return "", err
}
return key.String(), nil
}

2
swarm/api/storage_test.go
View File

@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
// to check put against the Api#Get
resp0 := testGet(t, api.api, bzzhash)
resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)
// check storage#Get

9
swarm/api/swarmfs_unix.go
View File

@ -91,11 +91,16 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
return nil, fmt.Errorf("%s is already mounted", cleanedMountPoint)
}
key, _, path, err := self.swarmApi.parseAndResolve(mhash, true)
uri, err := Parse("bzz:/" + mhash)
if err != nil {
return nil, fmt.Errorf("can't resolve %q: %v", mhash, err)
return nil, err
}
key, err := self.swarmApi.Resolve(uri)
if err != nil {
return nil, err
}
path := uri.Path
if len(path) > 0 {
path += "/"
}

96
swarm/api/uri.go Normal file
View File

@ -0,0 +1,96 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"fmt"
"net/url"
"strings"
)
// URI is a reference to content stored in swarm.
type URI struct {
// Scheme has one of the following values:
//
// * bzz - an entry in a swarm manifest
// * bzzr - raw swarm content
// * bzzi - immutable URI of an entry in a swarm manifest
// (address is not resolved)
Scheme string
// Addr is either a hexadecimal storage key or it an address which
// resolves to a storage key
Addr string
// Path is the path to the content within a swarm manifest
Path string
}
// Parse parses rawuri into a URI struct, where rawuri is expected to have one
// of the following formats:
//
// * <scheme>:/
// * <scheme>:/<addr>
// * <scheme>:/<addr>/<path>
// * <scheme>://
// * <scheme>://<addr>
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzzr or bzzi
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
return nil, err
}
uri := &URI{Scheme: u.Scheme}
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzzi", "bzzr":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
// handle URIs like bzz://<addr>/<path> where the addr and path
// have already been split by url.Parse
if u.Host != "" {
uri.Addr = u.Host
uri.Path = strings.TrimLeft(u.Path, "/")
return uri, nil
}
// URI is like bzz:/<addr>/<path> so split the addr and path from
// the raw path (which will be /<addr>/<path>)
parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
uri.Addr = parts[0]
if len(parts) == 2 {
uri.Path = parts[1]
}
return uri, nil
}
func (u *URI) Raw() bool {
return u.Scheme == "bzzr"
}
func (u *URI) Immutable() bool {
return u.Scheme == "bzzi"
}
func (u *URI) String() string {
return u.Scheme + ":/" + u.Addr + "/" + u.Path
}

120
swarm/api/uri_test.go Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"reflect"
"testing"
)
func TestParseURI(t *testing.T) {
type test struct {
uri string
expectURI *URI
expectErr bool
expectRaw bool
expectImmutable bool
}
tests := []test{
{
uri: "",
expectErr: true,
},
{
uri: "foo",
expectErr: true,
},
{
uri: "bzz",
expectErr: true,
},
{
uri: "bzz:",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
expectImmutable: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
expectRaw: true,
},
{
uri: "bzz:/",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz:/abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
expectRaw: true,
},
{
uri: "bzzr:/abc123",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
expectRaw: true,
},
{
uri: "bzzr:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
expectRaw: true,
},
{
uri: "bzz://",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz://abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz://abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
}
for _, x := range tests {
actual, err := Parse(x.uri)
if x.expectErr {
if err == nil {
t.Fatalf("expected %s to error", x.uri)
}
continue
}
if err != nil {
t.Fatalf("error parsing %s: %s", x.uri, err)
}
if !reflect.DeepEqual(actual, x.expectURI) {
t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
}
if actual.Raw() != x.expectRaw {
t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
}
if actual.Immutable() != x.expectImmutable {
t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
}
}
}

30
swarm/swarm.go
View File

@ -53,8 +53,8 @@ type Swarm struct {
privateKey *ecdsa.PrivateKey
corsString string
swapEnabled bool
lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
sfs *api.SwarmFS // need this to cleanup all the active mounts on node exit
lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
sfs *api.SwarmFS // need this to cleanup all the active mounts on node exit
}
type SwarmAPI struct {
@ -241,13 +241,6 @@ func (self *Swarm) Protocols() []p2p.Protocol {
func (self *Swarm) APIs() []rpc.API {
return []rpc.API{
// public APIs
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewStorage(self.api),
Public: true,
},
{
Namespace: "bzz",
Version: "0.1",
@ -255,11 +248,6 @@ func (self *Swarm) APIs() []rpc.API {
Public: true,
},
// admin APIs
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewFileSystem(self.api),
Public: false},
{
Namespace: "bzz",
Version: "0.1",
@ -278,6 +266,20 @@ func (self *Swarm) APIs() []rpc.API {
Service: self.sfs,
Public: false,
},
// storage APIs
// DEPRECATED: Use the HTTP API instead
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewStorage(self.api),
Public: true,
},
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewFileSystem(self.api),
Public: false,
},
// {Namespace, Version, api.NewAdmin(self), false},
}
}