Merge tag 'v0.4.23' into release

Release v0.4.23

.circleci/config.yml

@@ -1,8 +1,8 @@
-version: 2.0
+version: 2.1
 
 aliases:
   make_out_dirs: &make_out_dirs
-    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_WORKSPACE $CIRCLE_TEST_REPORTS/{unit,sharness}
+    run: mkdir -p /tmp/circleci-artifacts /tmp/circleci-workspace /tmp/circleci-test-results/{unit,sharness}
   restore_gomod: &restore_gomod
     restore_cache:
       keys:
@@ -17,28 +17,50 @@ aliases:
           - ~/go/pkg/mod
           - ~/.cache/go-build/
 
+default_environment: &default_environment
+  SERVICE: circle-ci
+  TRAVIS: 1
+  CIRCLE: 1
+  CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+  CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+  GIT_PAGER: cat
 
-defaults: &defaults
-  working_directory: ~/ipfs/go-ipfs
-  environment:
-    GIT_PAGER: cat
-    GO111MODULE: "on"
-    TEST_NO_DOCKER: 1
-    TEST_NO_FUSE: 1
-    GOPATH: /home/circleci/go
-    CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-    CIRCLE: 1
-    SERVICE: circle-ci
-    CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-    CIRCLE_WORKSPACE: /tmp/circleci-workspace
-    TEST_VERBOSE: 1
-    TRAVIS: 1
+executors:
+  golang:
+    parameters:
+      tag:
+        type: string
+        default: "1.12"
+    docker:
+      - image: circleci/golang:<< parameters.tag >>
+    working_directory: ~/ipfs/go-ipfs
+    environment:
+      <<: *default_environment
+      GO111MODULE: "on"
+      TEST_NO_DOCKER: 1
+      TEST_NO_FUSE: 1
+      GOPATH: /home/circleci/go
+      TEST_VERBOSE: 1
+  node:
+    docker:
+      - image: circleci/node:10
+    working_directory: ~/ipfs/go-ipfs
+    environment:
+      <<: *default_environment
+  node-browsers:
+    docker:
+      - image: circleci/node:12-browsers
+    working_directory: ~/ipfs/go-ipfs
+    environment:
+      <<: *default_environment
+      NO_SANDBOX: true
+      IPFS_REUSEPORT: false
+      LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
+      E2E_IPFSD_TYPE: go
 
 jobs:
   gobuild:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor: golang
     steps:
     - checkout
     - *make_out_dirs
@@ -53,9 +75,7 @@ jobs:
           TEST_NO_FUSE: 1
     - *store_gomod
   golint:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
+    executor: golang
     steps:
     - checkout
     - *make_out_dirs
@@ -64,9 +84,7 @@ jobs:
         make -O test_go_lint
     - *store_gomod
   gotest:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor: golang
     steps:
     - checkout
     - *make_out_dirs
@@ -93,9 +111,7 @@ jobs:
     - store_artifacts:
         path: /tmp/circleci-test-results
   sharness:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor: golang
     steps:
     - run: sudo apt install socat
     - checkout
@@ -122,9 +138,7 @@ jobs:
     - store_artifacts:
         path: /tmp/circleci-test-results
   build:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor: golang
     steps:
       - checkout
       - *make_out_dirs
@@ -143,9 +157,8 @@ jobs:
             - bin/ipfs
       - *store_gomod
   interop:
-    docker:
-      - image: circleci/node:10
-    <<: *defaults
+    executor: node
+    parallelism: 4
     steps:
       - *make_out_dirs
       - attach_workspace:
@@ -160,23 +173,43 @@ jobs:
             - v1-interop-{{ checksum "~/ipfs/go-ipfs/interop/package-lock.json" }}
             - v1-interop-
       - run:
-          command: npm install
+          name: Installing dependencies
+          command: |
+            npm install
           working_directory: ~/ipfs/go-ipfs/interop
-          environment:
-            IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
       - save_cache:
           key: v1-interop-{{ checksum "~/ipfs/go-ipfs/interop/package-lock.json" }}
           paths:
             - ~/ipfs/go-ipfs/interop/node_modules
       - run:
-          command: npm run test:node
+          name: Installing js-ipfs
+          command: |
+            npm install ipfs ipfs-http-client
+          working_directory: ~/ipfs/go-ipfs/interop
+      - run:
+          name: Installing reporting tools
+          command: |
+            npm install --save-dev [email protected] [email protected]
+          working_directory: ~/ipfs/go-ipfs/interop
+      - run:
+          name: Running tests
+          command: |
+            mkdir -p /tmp/test-results/interop/
+            export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)"
+            echo '{"reporterEnabled": "mocha-junit-reporter, spec"}' > mocha-conf.json
+            node_modules/.bin/mocha -R mocha-multi-reporters --reporter-options "configFile=mocha-conf.json" \
+              $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" test/node.js | circleci tests split)
           working_directory: ~/ipfs/go-ipfs/interop
           environment:
+            IPFS_REUSEPORT: false
+            LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
             IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
+      - store_test_results:
+          path: /tmp/test-results
   go-ipfs-api:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor:
+      name: golang
+      tag: "1.13"
     steps:
       - *make_out_dirs
       - attach_workspace:
@@ -192,7 +225,7 @@ jobs:
           background: true
       - run:
           name: Waiting for the daemon
-          no_output_timeout: 10s
+          no_output_timeout: 30s
           command: |
             while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do
               sleep 1
@@ -213,9 +246,7 @@ jobs:
           name: Stopping the daemon
           command: /tmp/circleci-workspace/bin/ipfs shutdown
   go-ipfs-http-client:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
+    executor: golang
     steps:
       - *make_out_dirs
       - attach_workspace:
@@ -240,6 +271,41 @@ jobs:
           paths:
             - ~/go/pkg/mod
             - ~/.cache/go-build/
+  ipfs-webui:
+    executor: node-browsers
+    steps:
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run:
+          name: Cloning
+          command: |
+            git clone https://github.com/ipfs-shipyard/ipfs-webui.git
+            git -C ipfs-webui log -1
+      - restore_cache:
+          keys:
+            - v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}
+            - v1-ipfs-webui-
+      - run:
+          name: Installing dependencies
+          command: |
+            npm install
+          working_directory: ~/ipfs/go-ipfs/ipfs-webui
+      - run:
+          name: Running upstream tests (finish early if they fail)
+          command: |
+            npm test || circleci-agent step halt
+          working_directory: ~/ipfs/go-ipfs/ipfs-webui
+      - run:
+          name: Running tests with go-ipfs built from current commit
+          command: npm test
+          working_directory: ~/ipfs/go-ipfs/ipfs-webui
+          environment:
+            IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
+      - save_cache:
+          key: v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}
+          paths:
+            - ~/ipfs/go-ipfs/ipfs-webui/node_modules
 workflows:
   version: 2
   test:
@@ -258,3 +324,6 @@ workflows:
     - go-ipfs-http-client:
         requires:
           - build
+    - ipfs-webui:
+        requires:
+          - build

core/coreapi/unixfs.go

@@ -3,6 +3,7 @@ package coreapi
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"github.com/ipfs/go-ipfs/core"
 
@@ -28,6 +29,28 @@ import (
 
 type UnixfsAPI CoreAPI
 
+var nilNode *core.IpfsNode
+var once sync.Once
+
+func getOrCreateNilNode() (*core.IpfsNode,error) {
+	once.Do(func() {
+		if nilNode != nil {
+			return
+		}
+		node, err := core.NewNode(context.Background(), &core.BuildCfg{
+			//TODO: need this to be true or all files
+			// hashed will be stored in memory!
+			NilRepo: true,
+		})
+		if err != nil {
+			panic(err)
+		}
+		nilNode = node
+	})
+
+	return nilNode, nil
+}
+
 // Add builds a merkledag node from a reader, adds it to the blockstore,
 // and returns the key representing that node.
 func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options.UnixfsAddOption) (path.Resolved, error) {
@@ -61,17 +84,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	pinning := api.pinning
 
 	if settings.OnlyHash {
-		nilnode, err := core.NewNode(ctx, &core.BuildCfg{
-			//TODO: need this to be true or all files
-			// hashed will be stored in memory!
-			NilRepo: true,
-		})
+		node, err := getOrCreateNilNode()
 		if err != nil {
 			return nil, err
 		}
-		addblockstore = nilnode.Blockstore
-		exch = nilnode.Exchange
-		pinning = nilnode.Pinning
+		addblockstore = node.Blockstore
+		exch = node.Exchange
+		pinning = node.Pinning
 	}
 
 	bserv := blockservice.New(addblockstore, exch) // hash security 001
@@ -127,8 +146,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 		return nil, err
 	}
 
-	if err := api.provider.Provide(nd.Cid()); err != nil {
-		return nil, err
+	if !settings.OnlyHash {
+		if err := api.provider.Provide(nd.Cid()); err != nil {
+			return nil, err
+		}
 	}
 
 	return path.IpfsPath(nd.Cid()), nil

core/corehttp/gateway_handler.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"net/url"
 	gopath "path"
+	"regexp"
 	"runtime/debug"
 	"strings"
 	"time"
@@ -151,6 +152,18 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		ipnsHostname = true
 	}
 
+	// Service Worker registration request
+	if r.Header.Get("Service-Worker") == "script" {
+		// Disallow Service Worker registration on namespace roots
+		// https://github.com/ipfs/go-ipfs/issues/4025
+		matched, _ := regexp.MatchString(`^/ip[fn]s/[^/]+$`, r.URL.Path)
+		if matched {
+			err := fmt.Errorf("registration is not allowed for this scope")
+			webError(w, "navigator.serviceWorker", err, http.StatusBadRequest)
+			return
+		}
+	}
+
 	parsedPath := ipath.New(urlPath)
 	if err := parsedPath.IsValid(); err != nil {
 		webError(w, "invalid ipfs path", err, http.StatusBadRequest)

core/corehttp/webui.go

@@ -1,7 +1,7 @@
 package corehttp
 
 // TODO: move to IPNS
-const WebUIPath = "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub"
+const WebUIPath = "/ipfs/Qmexhq2sBHnXQbvyP2GfUdbnY7HCagH2Mw5vUNSBn2nxip"
 
 // this is a list of all past webUI paths.
 var WebUIPaths = []string{
@@ -22,6 +22,17 @@ var WebUIPaths = []string{
 	"/ipfs/QmRyWyKWmphamkMRnJVjUTzSFSAAZowYP4rnbgnfMXC9Mr",
 	"/ipfs/QmU3o9bvfenhTKhxUakbYrLDnZU7HezAVxPM6Ehjw9Xjqy",
 	"/ipfs/QmPhnvn747LqwPYMJmQVorMaGbMSgA7mRRoyyZYz3DoZRQ",
+	"/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub",
+	"/ipfs/QmQNHd1suZTktPRhP7DD4nKWG46ZRSxkwHocycHVrK3dYW",
+	"/ipfs/QmNyMYhwJUS1cVvaWoVBhrW8KPj1qmie7rZcWo8f1Bvkhz",
+	"/ipfs/QmVTiRTQ72qiH4usAGT4c6qVxCMv4hFMUH9fvU6mktaXdP",
+	"/ipfs/QmYcP4sp1nraBiCYi6i9kqdaKobrK32yyMpTrM5JDA8a2C",
+	"/ipfs/QmUtMmxgHnDvQq4bpH6Y9MaLN1hpfjJz5LZcq941BEqEXs",
+	"/ipfs/QmPURAjo3oneGH53ovt68UZEBvsc8nNmEhQZEpsVEQUMZE",
+	"/ipfs/QmeSXt32frzhvewLKwA1dePTSjkTfGVwTh55ZcsJxrCSnk",
+	"/ipfs/QmcjeTciMNgEBe4xXvEaA4TQtwTRkXucx7DmKWViXSmX7m",
+	"/ipfs/QmfNbSskgvTXYhuqP8tb9AKbCkyRcCy3WeiXwD9y5LeoqK",
+	"/ipfs/QmPkojhjJkJ5LEGBDrAvdftrjAYmi9GU5Cq27mWvZTDieW",
 }
 
 var WebUIOption = RedirectOption("webui", WebUIPath)