From c786f75389121c29d4380c53a013759254457776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 19 Dec 2017 09:49:30 +0100 Subject: [PATCH] swarm: bzz-list, bzz-raw and bzz-immutable schemes (#15667) * swarm/api: url scheme bzz-list for getting list of files from manifest Replace query parameter list=true for listing all files contained in a swarm manifest with a new URL scheme bzz-list. * swarm: replaace bzzr and bzzi schemes with bzz-raw and bzz-immutable New URI Shemes are added and old ones are deprecated, but not removed. Old Schemes bzzr and bzzi are functional for backward compatibility. * swarm/api: completely remove bzzr and bzzi schemes Remove old schemes in favour of bzz-raw and bzz-immutable. * swarm/api: revert "completely remove bzzr and bzzi schemes" Keep bzzr and bzzi schemes for backward compatibility. At least until 0.3 swarm release. --- swarm/api/api.go | 2 +- swarm/api/api_test.go | 2 +- swarm/api/client/client.go | 6 +- swarm/api/http/roundtripper.go | 4 +- swarm/api/http/server.go | 37 ++++++----- swarm/api/http/server_test.go | 98 +++++++++++++++++++++++++++-- swarm/api/http/templates.go | 2 +- swarm/api/uri.go | 27 ++++++-- swarm/api/uri_test.go | 72 ++++++++++++++++----- swarm/dev/scripts/random-uploads.sh | 2 +- 10 files changed, 203 insertions(+), 49 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 79de29a1c..8c4bca2ec 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // if the URI is immutable, check if the address is a hash isHash := hashMatcher.MatchString(uri.Addr) - if uri.Immutable() { + if uri.Immutable() || uri.DeprecatedImmutable() { if !isHash { return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr) } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index f9caed27f..e673f76c4 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) { api := &Api{dns: x.dns} uri := &URI{Addr: x.addr, Scheme: "bzz"} if x.immutable { - uri.Scheme = "bzzi" + uri.Scheme = "bzz-immutable" } res, err := api.Resolve(uri) if err == nil { diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 7952d3fb6..8165d52d7 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { if size <= 0 { return "", errors.New("data size must be greater than zero") } - req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r) + req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r) if err != nil { return "", err } @@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { // DownloadRaw downloads raw data from swarm func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) { - uri := c.Gateway + "/bzzr:/" + hash + uri := c.Gateway + "/bzz-raw:/" + hash res, err := http.DefaultClient.Get(uri) if err != nil { return nil, err @@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) { // // where entries ending with "/" are common prefixes. func (c *Client) List(hash, prefix string) (*api.ManifestList, error) { - res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true") + res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix) if err != nil { return nil, err } diff --git a/swarm/api/http/roundtripper.go b/swarm/api/http/roundtripper.go index 328177a21..019431771 100644 --- a/swarm/api/http/roundtripper.go +++ b/swarm/api/http/roundtripper.go @@ -35,8 +35,8 @@ import ( client := httpclient.New() // for (private) swarm proxy running locally client.RegisterScheme("bzz", &http.RoundTripper{Port: port}) -client.RegisterScheme("bzzi", &http.RoundTripper{Port: port}) -client.RegisterScheme("bzzr", &http.RoundTripper{Port: port}) +client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port}) +client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port}) The port you give the Roundtripper is the port the swarm proxy is listening on. If Host is left empty, localhost is assumed. diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 65f6afab7..3872cbc4f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -86,7 +86,7 @@ type Request struct { uri *api.URI } -// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request +// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { @@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { fmt.Fprint(w, newKey) } -// HandleGetRaw handles a GET request to bzzr:// and responds with +// HandleGetRaw handles a GET request to bzz-raw:// and responds with // the raw content stored at the given storage key func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) @@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { } } -// HandleGetList handles a GET request to bzz:// which has -// the "list" query parameter set to "true" and returns a list of all files -// contained in under grouped into common prefixes using -// "/" as a delimiter +// HandleGetList handles a GET request to bzz-list:// and returns +// a list of all files contained in under grouped into +// common prefixes using "/" as a delimiter func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently) + http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) return } @@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { if strings.Contains(r.Header.Get("Accept"), "text/html") { w.Header().Set("Content-Type", "text/html") err := htmlListTemplate.Execute(w, &htmlListData{ - URI: r.uri, + URI: &api.URI{ + Scheme: "bzz", + Addr: r.uri.Addr, + Path: r.uri.Path, + }, List: &list, }) if err != nil { @@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.Method { case "POST": - if uri.Raw() { + if uri.Raw() || uri.DeprecatedRaw() { s.HandlePostRaw(w, req) } else { s.HandlePostFiles(w, req) @@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // new manifest leaving the existing one intact, so it isn't // strictly a traditional PUT request which replaces content // at a URI, and POST is more ubiquitous) - if uri.Raw() { + if uri.Raw() || uri.DeprecatedRaw() { ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return } else { @@ -609,28 +612,28 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } case "DELETE": - if uri.Raw() { + if uri.Raw() || uri.DeprecatedRaw() { ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) return } s.HandleDelete(w, req) case "GET": - if uri.Raw() { + if uri.Raw() || uri.DeprecatedRaw() { s.HandleGetRaw(w, req) return } + if uri.List() { + s.HandleGetList(w, req) + return + } + if r.Header.Get("Accept") == "application/x-tar" { s.HandleGetFiles(w, req) return } - if r.URL.Query().Get("list") == "true" { - s.HandleGetList(w, req) - return - } - s.HandleGetFile(w, req) default: diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index ffeaf6e0d..dbbfa3b07 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) { wg.Wait() } - _, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a") + _, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a") if err != nil { t.Fatalf("Failed to connect to proxy: %v", err) } @@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) { var resp *http.Response var respbody []byte - url := srv.URL + "/bzzr:/" + url := srv.URL + "/bzz-raw:/" if k[:] != "" { url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" } @@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) { } } + for _, c := range []struct { + path string + json string + html string + }{ + { + path: "/", + json: `{"common_prefixes":["a/"]}`, + html: "\n\n\n \n \n Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n
PathTypeSize
a/DIR-
\n
\n\n", + }, + { + path: "/a/", + json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`, + html: "\n\n\n \n \n Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b/DIR-
a0
\n
\n\n", + }, + { + path: "/a/b/", + json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`, + html: "\n\n\n \n \n Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\n \n\t\n\t \n\t \n\t \n\t\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b0
c0
\n
\n\n", + }, + { + path: "/x", + }, + { + path: "", + }, + } { + k := c.path + url := srv.URL + "/bzz-list:/" + if k[:] != "" { + url += common.ToHex(key[0])[2:] + "/" + k[1:] + } + t.Run("json list "+c.path, func(t *testing.T) { + resp, err := http.Get(url) + if err != nil { + t.Fatalf("HTTP request: %v", err) + } + defer resp.Body.Close() + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Read response body: %v", err) + } + + body := strings.TrimSpace(string(respbody)) + if body != c.json { + isexpectedfailrequest := false + + for _, r := range expectedfailrequests { + if k[:] == r { + isexpectedfailrequest = true + } + } + if !isexpectedfailrequest { + t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body) + } + } + }) + t.Run("html list "+c.path, func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatalf("New request: %v", err) + } + req.Header.Set("Accept", "text/html") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("HTTP request: %v", err) + } + defer resp.Body.Close() + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Read response body: %v", err) + } + + if string(respbody) != c.html { + isexpectedfailrequest := false + + for _, r := range expectedfailrequests { + if k[:] == r { + isexpectedfailrequest = true + } + } + if !isexpectedfailrequest { + t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody)) + } + } + }) + } + nonhashtests := []string{ srv.URL + "/bzz:/name", - srv.URL + "/bzzi:/nonhash", - srv.URL + "/bzzr:/nonhash", + srv.URL + "/bzz-immutable:/nonhash", + srv.URL + "/bzz-raw:/nonhash", + srv.URL + "/bzz-list:/nonhash", } nonhashresponses := []string{ "error resolving name: no DNS to resolve name: "name"", "error resolving nonhash: immutable address not a content hash: "nonhash"", "error resolving nonhash: no DNS to resolve name: "nonhash"", + "error resolving nonhash: no DNS to resolve name: "nonhash"", } for i, url := range nonhashtests { diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 9ae434a7e..53ce7b5a2 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu {{ range .List.CommonPrefixes }} - {{ basename . }}/ + {{ basename . }}/ DIR - diff --git a/swarm/api/uri.go b/swarm/api/uri.go index caed4212d..af1dc7445 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -26,7 +26,13 @@ import ( type URI struct { // Scheme has one of the following values: // - // * bzz - an entry in a swarm manifest + // * bzz - an entry in a swarm manifest + // * bzz-raw - raw swarm content + // * bzz-immutable - immutable URI of an entry in a swarm manifest + // (address is not resolved) + // * bzz-list - list of all files contained in a swarm manifest + // + // Deprecated Schemes: // * bzzr - raw swarm content // * bzzi - immutable URI of an entry in a swarm manifest // (address is not resolved) @@ -50,7 +56,8 @@ type URI struct { // * :// // * :/// // -// with scheme one of bzz, bzzr or bzzi +// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list +// or deprecated ones bzzr and bzzi func Parse(rawuri string) (*URI, error) { u, err := url.Parse(rawuri) if err != nil { @@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzzi", "bzzr": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) { } func (u *URI) Raw() bool { - return u.Scheme == "bzzr" + return u.Scheme == "bzz-raw" } func (u *URI) Immutable() bool { + return u.Scheme == "bzz-immutable" +} + +func (u *URI) List() bool { + return u.Scheme == "bzz-list" +} + +func (u *URI) DeprecatedRaw() bool { + return u.Scheme == "bzzr" +} + +func (u *URI) DeprecatedImmutable() bool { return u.Scheme == "bzzi" } diff --git a/swarm/api/uri_test.go b/swarm/api/uri_test.go index 7d4160601..babb2834e 100644 --- a/swarm/api/uri_test.go +++ b/swarm/api/uri_test.go @@ -23,11 +23,14 @@ import ( func TestParseURI(t *testing.T) { type test struct { - uri string - expectURI *URI - expectErr bool - expectRaw bool - expectImmutable bool + uri string + expectURI *URI + expectErr bool + expectRaw bool + expectImmutable bool + expectList bool + expectDeprecatedRaw bool + expectDeprecatedImmutable bool } tests := []test{ { @@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) { expectURI: &URI{Scheme: "bzz"}, }, { - uri: "bzzi:", - expectURI: &URI{Scheme: "bzzi"}, + uri: "bzz-immutable:", + expectURI: &URI{Scheme: "bzz-immutable"}, expectImmutable: true, }, { - uri: "bzzr:", - expectURI: &URI{Scheme: "bzzr"}, + uri: "bzz-raw:", + expectURI: &URI{Scheme: "bzz-raw"}, expectRaw: true, }, { @@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) { expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, }, { - uri: "bzzr:/", - expectURI: &URI{Scheme: "bzzr"}, + uri: "bzz-raw:/", + expectURI: &URI{Scheme: "bzz-raw"}, expectRaw: true, }, { - uri: "bzzr:/abc123", - expectURI: &URI{Scheme: "bzzr", Addr: "abc123"}, + uri: "bzz-raw:/abc123", + expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"}, expectRaw: true, }, { - uri: "bzzr:/abc123/path/to/entry", - expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"}, + uri: "bzz-raw:/abc123/path/to/entry", + expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"}, expectRaw: true, }, { @@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) { uri: "bzz://abc123/path/to/entry", expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, }, + { + uri: "bzz-list:", + expectURI: &URI{Scheme: "bzz-list"}, + expectList: true, + }, + { + uri: "bzz-list:/", + expectURI: &URI{Scheme: "bzz-list"}, + expectList: true, + }, + { + uri: "bzzr:", + expectURI: &URI{Scheme: "bzzr"}, + expectDeprecatedRaw: true, + }, + { + uri: "bzzr:/", + expectURI: &URI{Scheme: "bzzr"}, + expectDeprecatedRaw: true, + }, + { + uri: "bzzi:", + expectURI: &URI{Scheme: "bzzi"}, + expectDeprecatedImmutable: true, + }, + { + uri: "bzzi:/", + expectURI: &URI{Scheme: "bzzi"}, + expectDeprecatedImmutable: true, + }, } for _, x := range tests { actual, err := Parse(x.uri) @@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) { if actual.Immutable() != x.expectImmutable { t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable()) } + if actual.List() != x.expectList { + t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List()) + } + if actual.DeprecatedRaw() != x.expectDeprecatedRaw { + t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw()) + } + if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable { + t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable()) + } } } diff --git a/swarm/dev/scripts/random-uploads.sh b/swarm/dev/scripts/random-uploads.sh index db7887e3c..563a51bef 100755 --- a/swarm/dev/scripts/random-uploads.sh +++ b/swarm/dev/scripts/random-uploads.sh @@ -46,7 +46,7 @@ main() { } do_random_upload() { - curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/" + curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/" } random_data() {