poanetwork
/
quorum
mirror of https://github.com/poanetwork/quorum.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

swarm: bzz-list, bzz-raw and bzz-immutable schemes (#15667) 

* swarm/api: url scheme bzz-list for getting list of files from manifest

Replace query parameter list=true for listing all files contained
in a swarm manifest with a new URL scheme bzz-list.

* swarm: replaace bzzr and bzzi schemes with bzz-raw and bzz-immutable

New URI Shemes are added and old ones are deprecated, but not removed.
Old Schemes bzzr and bzzi are functional for backward compatibility.

* swarm/api: completely remove bzzr and bzzi schemes

Remove old schemes in favour of bzz-raw and
bzz-immutable.

* swarm/api: revert "completely remove bzzr and bzzi schemes"

Keep bzzr and bzzi schemes for backward compatibility. At least
until 0.3 swarm release.
This commit is contained in:
Janoš Guljaš 2017-12-19 09:49:30 +01:00 committed by Péter Szilágyi
parent 7f9d94fe9a
commit c786f75389
10 changed files with 203 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
swarm/api/api.go
View File

@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
// if the URI is immutable, check if the address is a hash
isHash := hashMatcher.MatchString(uri.Addr)
if uri.Immutable() {
if uri.Immutable() || uri.DeprecatedImmutable() {
if !isHash {
return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
}

2
swarm/api/api_test.go
View File

@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) {
api := &Api{dns: x.dns}
uri := &URI{Addr: x.addr, Scheme: "bzz"}
if x.immutable {
uri.Scheme = "bzzi"
uri.Scheme = "bzz-immutable"
}
res, err := api.Resolve(uri)
if err == nil {

6
swarm/api/client/client.go
View File

@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
if err != nil {
return "", err
}
@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
// DownloadRaw downloads raw data from swarm
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
uri := c.Gateway + "/bzzr:/" + hash
uri := c.Gateway + "/bzz-raw:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
if err != nil {
return nil, err
}

4
swarm/api/http/roundtripper.go
View File

@ -35,8 +35,8 @@ import (
client := httpclient.New()
// for (private) swarm proxy running locally
client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
The port you give the Roundtripper is the port the swarm proxy is listening on.
If Host is left empty, localhost is assumed.

37
swarm/api/http/server.go
View File

@ -86,7 +86,7 @@ type Request struct {
uri *api.URI
}
// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
fmt.Fprint(w, newKey)
}
// HandleGetRaw handles a GET request to bzzr://<key> and responds with
// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
// the raw content stored at the given storage key
func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
}
}
// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
// the "list" query parameter set to "true" and returns a list of all files
// contained in <manifest> under <path> grouped into common prefixes using
// "/" as a delimiter
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
URI: r.uri,
URI: &api.URI{
Scheme: "bzz",
Addr: r.uri.Addr,
Path: r.uri.Path,
},
List: &list,
})
if err != nil {
@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "POST":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
s.HandlePostRaw(w, req)
} else {
s.HandlePostFiles(w, req)
@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// new manifest leaving the existing one intact, so it isn't
// strictly a traditional PUT request which replaces content
// at a URI, and POST is more ubiquitous)
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
} else {
@ -609,28 +612,28 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
case "DELETE":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
return
}
s.HandleDelete(w, req)
case "GET":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
s.HandleGetRaw(w, req)
return
}
if uri.List() {
s.HandleGetList(w, req)
return
}
if r.Header.Get("Accept") == "application/x-tar" {
s.HandleGetFiles(w, req)
return
}
if r.URL.Query().Get("list") == "true" {
s.HandleGetList(w, req)
return
}
s.HandleGetFile(w, req)
default:

98
swarm/api/http/server_test.go
View File

@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
wg.Wait()
}
_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
if err != nil {
t.Fatalf("Failed to connect to proxy: %v", err)
}
@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzzr:/"
url := srv.URL + "/bzz-raw:/"
if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
}
@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
}
}
for _, c := range []struct {
path string
json string
html string
}{
{
path: "/",
json: `{"common_prefixes":["a/"]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\t<tr>\n\t <td><a href=\"a/\">a/</a></td>\n\t <td>DIR</td>\n\t <td>-</td>\n\t</tr>\n \n\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/a/",
json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\t<tr>\n\t <td><a href=\"b/\">b/</a></td>\n\t <td>DIR</td>\n\t <td>-</td>\n\t</tr>\n \n\n \n\t<tr>\n\t <td><a href=\"a\">a</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/a/b/",
json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\n \n\t<tr>\n\t <td><a href=\"b\">b</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n\t<tr>\n\t <td><a href=\"c\">c</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/x",
},
{
path: "",
},
} {
k := c.path
url := srv.URL + "/bzz-list:/"
if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:]
}
t.Run("json list "+c.path, func(t *testing.T) {
resp, err := http.Get(url)
if err != nil {
t.Fatalf("HTTP request: %v", err)
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Read response body: %v", err)
}
body := strings.TrimSpace(string(respbody))
if body != c.json {
isexpectedfailrequest := false
for _, r := range expectedfailrequests {
if k[:] == r {
isexpectedfailrequest = true
}
}
if !isexpectedfailrequest {
t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
}
}
})
t.Run("html list "+c.path, func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatalf("New request: %v", err)
}
req.Header.Set("Accept", "text/html")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("HTTP request: %v", err)
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Read response body: %v", err)
}
if string(respbody) != c.html {
isexpectedfailrequest := false
for _, r := range expectedfailrequests {
if k[:] == r {
isexpectedfailrequest = true
}
}
if !isexpectedfailrequest {
t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
}
}
})
}
nonhashtests := []string{
srv.URL + "/bzz:/name",
srv.URL + "/bzzi:/nonhash",
srv.URL + "/bzzr:/nonhash",
srv.URL + "/bzz-immutable:/nonhash",
srv.URL + "/bzz-raw:/nonhash",
srv.URL + "/bzz-list:/nonhash",
}
nonhashresponses := []string{
"error resolving name: no DNS to resolve name: &#34;name&#34;",
"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
}
for i, url := range nonhashtests {

2
swarm/api/http/templates.go
View File

@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
<tbody>
{{ range .List.CommonPrefixes }}
<tr>
<td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
<td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
<td>DIR</td>
<td>-</td>
</tr>

27
swarm/api/uri.go
View File

@ -26,7 +26,13 @@ import (
type URI struct {
// Scheme has one of the following values:
//
// * bzz - an entry in a swarm manifest
// * bzz - an entry in a swarm manifest
// * bzz-raw - raw swarm content
// * bzz-immutable - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-list - list of all files contained in a swarm manifest
//
// Deprecated Schemes:
// * bzzr - raw swarm content
// * bzzi - immutable URI of an entry in a swarm manifest
// (address is not resolved)
@ -50,7 +56,8 @@ type URI struct {
// * <scheme>://<addr>
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzzr or bzzi
// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list
// or deprecated ones bzzr and bzzi
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) {
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzzi", "bzzr":
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) {
}
func (u *URI) Raw() bool {
return u.Scheme == "bzzr"
return u.Scheme == "bzz-raw"
}
func (u *URI) Immutable() bool {
return u.Scheme == "bzz-immutable"
}
func (u *URI) List() bool {
return u.Scheme == "bzz-list"
}
func (u *URI) DeprecatedRaw() bool {
return u.Scheme == "bzzr"
}
func (u *URI) DeprecatedImmutable() bool {
return u.Scheme == "bzzi"
}

72
swarm/api/uri_test.go
View File

@ -23,11 +23,14 @@ import (
func TestParseURI(t *testing.T) {
type test struct {
uri string
expectURI *URI
expectErr bool
expectRaw bool
expectImmutable bool
uri string
expectURI *URI
expectErr bool
expectRaw bool
expectImmutable bool
expectList bool
expectDeprecatedRaw bool
expectDeprecatedImmutable bool
}
tests := []test{
{
@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) {
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
uri: "bzz-immutable:",
expectURI: &URI{Scheme: "bzz-immutable"},
expectImmutable: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
uri: "bzz-raw:",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) {
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
uri: "bzz-raw:/",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzzr:/abc123",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
uri: "bzz-raw:/abc123",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
expectRaw: true,
},
{
uri: "bzzr:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
uri: "bzz-raw:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
expectRaw: true,
},
{
@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) {
uri: "bzz://abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-list:",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzz-list:/",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
{
uri: "bzzi:/",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
}
for _, x := range tests {
actual, err := Parse(x.uri)
@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) {
if actual.Immutable() != x.expectImmutable {
t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
}
if actual.List() != x.expectList {
t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
}
if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
}
if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
}
}
}

2
swarm/dev/scripts/random-uploads.sh
View File

@ -46,7 +46,7 @@ main() {
}
do_random_upload() {
curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/"
curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
}
random_data() {