swarm: bzz-list, bzz-raw and bzz-immutable schemes (#15667)

* swarm/api: url scheme bzz-list for getting list of files from manifest

Replace query parameter list=true for listing all files contained
in a swarm manifest with a new URL scheme bzz-list.

* swarm: replaace bzzr and bzzi schemes with bzz-raw and bzz-immutable

New URI Shemes are added and old ones are deprecated, but not removed.
Old Schemes bzzr and bzzi are functional for backward compatibility.

* swarm/api: completely remove bzzr and bzzi schemes

Remove old schemes in favour of bzz-raw and
bzz-immutable.

* swarm/api: revert "completely remove bzzr and bzzi schemes"

Keep bzzr and bzzi schemes for backward compatibility. At least
until 0.3 swarm release.
This commit is contained in:
Janoš Guljaš 2017-12-19 09:49:30 +01:00 committed by Péter Szilágyi
parent 7f9d94fe9a
commit c786f75389
10 changed files with 203 additions and 49 deletions

View File

@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
// if the URI is immutable, check if the address is a hash
isHash := hashMatcher.MatchString(uri.Addr)
if uri.Immutable() {
if uri.Immutable() || uri.DeprecatedImmutable() {
if !isHash {
return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
}

View File

@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) {
api := &Api{dns: x.dns}
uri := &URI{Addr: x.addr, Scheme: "bzz"}
if x.immutable {
uri.Scheme = "bzzi"
uri.Scheme = "bzz-immutable"
}
res, err := api.Resolve(uri)
if err == nil {

View File

@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
if err != nil {
return "", err
}
@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
// DownloadRaw downloads raw data from swarm
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
uri := c.Gateway + "/bzzr:/" + hash
uri := c.Gateway + "/bzz-raw:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
if err != nil {
return nil, err
}

View File

@ -35,8 +35,8 @@ import (
client := httpclient.New()
// for (private) swarm proxy running locally
client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
The port you give the Roundtripper is the port the swarm proxy is listening on.
If Host is left empty, localhost is assumed.

View File

@ -86,7 +86,7 @@ type Request struct {
uri *api.URI
}
// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
fmt.Fprint(w, newKey)
}
// HandleGetRaw handles a GET request to bzzr://<key> and responds with
// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
// the raw content stored at the given storage key
func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
}
}
// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
// the "list" query parameter set to "true" and returns a list of all files
// contained in <manifest> under <path> grouped into common prefixes using
// "/" as a delimiter
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
URI: r.uri,
URI: &api.URI{
Scheme: "bzz",
Addr: r.uri.Addr,
Path: r.uri.Path,
},
List: &list,
})
if err != nil {
@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "POST":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
s.HandlePostRaw(w, req)
} else {
s.HandlePostFiles(w, req)
@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// new manifest leaving the existing one intact, so it isn't
// strictly a traditional PUT request which replaces content
// at a URI, and POST is more ubiquitous)
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
} else {
@ -609,28 +612,28 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
case "DELETE":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
return
}
s.HandleDelete(w, req)
case "GET":
if uri.Raw() {
if uri.Raw() || uri.DeprecatedRaw() {
s.HandleGetRaw(w, req)
return
}
if uri.List() {
s.HandleGetList(w, req)
return
}
if r.Header.Get("Accept") == "application/x-tar" {
s.HandleGetFiles(w, req)
return
}
if r.URL.Query().Get("list") == "true" {
s.HandleGetList(w, req)
return
}
s.HandleGetFile(w, req)
default:

View File

@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
wg.Wait()
}
_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
if err != nil {
t.Fatalf("Failed to connect to proxy: %v", err)
}
@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzzr:/"
url := srv.URL + "/bzz-raw:/"
if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
}
@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
}
}
for _, c := range []struct {
path string
json string
html string
}{
{
path: "/",
json: `{"common_prefixes":["a/"]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\t<tr>\n\t <td><a href=\"a/\">a/</a></td>\n\t <td>DIR</td>\n\t <td>-</td>\n\t</tr>\n \n\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/a/",
json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\t<tr>\n\t <td><a href=\"b/\">b/</a></td>\n\t <td>DIR</td>\n\t <td>-</td>\n\t</tr>\n \n\n \n\t<tr>\n\t <td><a href=\"a\">a</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/a/b/",
json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
html: "<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n <hr>\n <table>\n <thead>\n <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n </tr>\n </thead>\n\n <tbody>\n \n\n \n\t<tr>\n\t <td><a href=\"b\">b</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n\t<tr>\n\t <td><a href=\"c\">c</a></td>\n\t <td></td>\n\t <td>0</td>\n\t</tr>\n \n </table>\n <hr>\n</body>\n",
},
{
path: "/x",
},
{
path: "",
},
} {
k := c.path
url := srv.URL + "/bzz-list:/"
if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:]
}
t.Run("json list "+c.path, func(t *testing.T) {
resp, err := http.Get(url)
if err != nil {
t.Fatalf("HTTP request: %v", err)
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Read response body: %v", err)
}
body := strings.TrimSpace(string(respbody))
if body != c.json {
isexpectedfailrequest := false
for _, r := range expectedfailrequests {
if k[:] == r {
isexpectedfailrequest = true
}
}
if !isexpectedfailrequest {
t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
}
}
})
t.Run("html list "+c.path, func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatalf("New request: %v", err)
}
req.Header.Set("Accept", "text/html")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("HTTP request: %v", err)
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Read response body: %v", err)
}
if string(respbody) != c.html {
isexpectedfailrequest := false
for _, r := range expectedfailrequests {
if k[:] == r {
isexpectedfailrequest = true
}
}
if !isexpectedfailrequest {
t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
}
}
})
}
nonhashtests := []string{
srv.URL + "/bzz:/name",
srv.URL + "/bzzi:/nonhash",
srv.URL + "/bzzr:/nonhash",
srv.URL + "/bzz-immutable:/nonhash",
srv.URL + "/bzz-raw:/nonhash",
srv.URL + "/bzz-list:/nonhash",
}
nonhashresponses := []string{
"error resolving name: no DNS to resolve name: &#34;name&#34;",
"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
}
for i, url := range nonhashtests {

View File

@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
<tbody>
{{ range .List.CommonPrefixes }}
<tr>
<td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
<td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
<td>DIR</td>
<td>-</td>
</tr>

View File

@ -27,6 +27,12 @@ type URI struct {
// Scheme has one of the following values:
//
// * bzz - an entry in a swarm manifest
// * bzz-raw - raw swarm content
// * bzz-immutable - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-list - list of all files contained in a swarm manifest
//
// Deprecated Schemes:
// * bzzr - raw swarm content
// * bzzi - immutable URI of an entry in a swarm manifest
// (address is not resolved)
@ -50,7 +56,8 @@ type URI struct {
// * <scheme>://<addr>
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzzr or bzzi
// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list
// or deprecated ones bzzr and bzzi
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) {
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzzi", "bzzr":
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) {
}
func (u *URI) Raw() bool {
return u.Scheme == "bzzr"
return u.Scheme == "bzz-raw"
}
func (u *URI) Immutable() bool {
return u.Scheme == "bzz-immutable"
}
func (u *URI) List() bool {
return u.Scheme == "bzz-list"
}
func (u *URI) DeprecatedRaw() bool {
return u.Scheme == "bzzr"
}
func (u *URI) DeprecatedImmutable() bool {
return u.Scheme == "bzzi"
}

View File

@ -28,6 +28,9 @@ func TestParseURI(t *testing.T) {
expectErr bool
expectRaw bool
expectImmutable bool
expectList bool
expectDeprecatedRaw bool
expectDeprecatedImmutable bool
}
tests := []test{
{
@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) {
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
uri: "bzz-immutable:",
expectURI: &URI{Scheme: "bzz-immutable"},
expectImmutable: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
uri: "bzz-raw:",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) {
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
uri: "bzz-raw:/",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzzr:/abc123",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
uri: "bzz-raw:/abc123",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
expectRaw: true,
},
{
uri: "bzzr:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
uri: "bzz-raw:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
expectRaw: true,
},
{
@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) {
uri: "bzz://abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-list:",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzz-list:/",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
{
uri: "bzzi:/",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
}
for _, x := range tests {
actual, err := Parse(x.uri)
@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) {
if actual.Immutable() != x.expectImmutable {
t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
}
if actual.List() != x.expectList {
t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
}
if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
}
if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
}
}
}

View File

@ -46,7 +46,7 @@ main() {
}
do_random_upload() {
curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/"
curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
}
random_data() {