add referrers API, ORAS artifact manifest support

Signed-off-by: Aviral Takkar <[email protected]>
oras-project · Aug 24, 2021 · 36ab5fc · 36ab5fc
1 parent 2cd47c1
commit 36ab5fc
Show file tree

Hide file tree

Showing 23 changed files with 1,204 additions and 1 deletion.
diff --git a/docs/referrers.md b/docs/referrers.md
@@ -0,0 +1,235 @@
+[[__TOC__]]
+
+# ORAS Artifacts Distribution
+
+This document describes an experimental prototype that implements the
+[ORAS Artifact Manifest](https://github.com/oras-project/artifacts-spec) spec.
+
+## Usage - Push, Discover, Pull
+
+The following steps illustrate how ORAS artifacts can be stored and retrieved from a registry.
+The artifact in this example is a Notary V2 [signature](https://github.com/notaryproject/nv2/tree/prototype-2/docs/nv2).
+
+### Prerequisites
+
+- Local registry prototype instance
+- [docker-generate](https://github.com/shizhMSFT/docker-generate)
+- [nv2](https://github.com/notaryproject/nv2)
+- `curl`
+- `jq`
+
+### Push an image to your registry
+
+```shell
+# Initialize local registry variables
+regIp="127.0.0.1" && \
+  regPort="5000" && \
+  registry="$regIp:$regPort" && \
+  repo="busybox" && \
+  tag="latest" && \
+  image="$repo:$tag" && \
+  reference="$registry/$image"
+
+# Pull an image from docker hub and push to local registry
+docker pull $image && \
+  docker tag $image $reference && \
+  docker push $reference
+```
+
+### Generate image manifest and sign it
+
+```shell
+# Generate self-signed certificates
+openssl req \
+  -x509 \
+  -sha256 \
+  -nodes \
+  -newkey rsa:2048 \
+  -days 365 \
+  -subj "/CN=$regIp/O=example inc/C=IN/ST=Haryana/L=Gurgaon" \
+  -addext "subjectAltName=IP:$regIp" \
+  -keyout example.key \
+  -out example.crt
+
+# Generate image manifest
+manifestFile="manifest-to-sign.json" && \
+  docker generate manifest $image > $manifestFile
+
+# Sign manifest
+signatureFile="manifest-signature.jwt" && \
+  nv2 sign --method x509 \
+    -k example.key \
+    -c example.crt \
+    -r $reference \
+    -o $signatureFile \
+    file:$manifestFile
+```
+
+### Obtain manifest and signature digests
+
+```shell
+manifestDigest="sha256:$(sha256sum $manifestFile | cut -d " " -f 1)" && \
+  signatureDigest="sha256:$(sha256sum $signatureFile | cut -d " " -f 1)"
+```
+
+### Create an Artifact file referencing the manifest that was signed and its signature as blob
+
+```shell
+artifactFile="artifact.json" && \
+  artifactMediaType="application/vnd.cncf.oras.artifact.manifest.v1+json" && \
+  artifactType="application/vnd.cncf.notary.v2" && \
+  signatureMediaType="application/vnd.cncf.notary.signature.v2+jwt" && \
+  signatureFileSize=`wc -c < $signatureFile` && \
+  manifestMediaType="$(cat $manifestFile | jq -r '.mediaType')" && \
+  manifestFileSize=`wc -c < $manifestFile`
+
+cat <<EOF > $artifactFile
+{
+  "mediaType": "$artifactMediaType",
+  "artifactType": "$artifactType",
+  "blobs": [
+    {
+      "mediaType": "$signatureMediaType",
+      "digest": "$signatureDigest",
+      "size": $signatureFileSize
+    }
+  ],
+  "subjectManifest": {
+      "mediaType": "$manifestMediaType",
+      "digest": "$manifestDigest",
+      "size": $manifestFileSize
+  }
+}
+EOF
+```
+
+### Obtain artifact digest
+
+```shell
+artifactDigest="sha256:$(sha256sum $artifactFile | cut -d " " -f 1)"
+```
+
+### Push signature and artifact
+
+```shell
+# Initiate blob upload and obtain PUT location
+blobPutLocation=`curl -I -X POST -s http://$registry/v2/$repo/blobs/uploads/ | grep "Location: " | sed -e "s/Location: //;s/$/\&digest=$signatureDigest/;s/\r//"`
+
+# Push signature blob
+curl -X PUT -H "Content-Type: application/octet-stream" --data-binary @"$signatureFile" $blobPutLocation
+
+# Push artifact
+curl -X PUT --data-binary @"$artifactFile" -H "Content-Type: $artifactMediaType" "http://$registry/v2/$repo/manifests/$artifactDigest"
+```
+
+### List referrers
+
+```shell
+# Retrieve referrers
+curl -s "http://$registry/oras/artifacts/v1/$repo/manifests/$manifestDigest/referrers?artifactType=$artifactType" | jq
+```
+
+### Verify signature
+
+```shell
+# Retrieve signature
+artifactDigest=`curl -s "http://$registry/oras/artifacts/v1/$repo/manifests/$manifestDigest/referrers?artifactType=$artifactType" | jq -r '.references[0].digest'` && \
+  signatureDigest=`curl -s "http://$registry/oras/artifacts/v1/$repo/manifests/$artifactDigest" | jq -r '.blobs[0].digest'` && \
+  retrievedSignatureFile="retrieved-signature.json" && \
+  curl -s http://$registry/v2/$repo/blobs/$signatureDigest > $retrievedSignatureFile
+
+# Verify signature
+nv2 verify \
+  -f $retrievedSignatureFile \
+  -c example.crt \
+  file:$manifestFile
+```
+
+## Implementation
+
+To power the [/referrers](https://github.com/oras-project/artifacts-spec/blob/main/manifest-referrers-api.md) API, the
+referrers of a manifest are indexed in the repository store. The following example illustrates the creation of this
+index.
+
+The `nginx:v1` image is already persisted:
+
+- repository: `nginx`
+- digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m`
+- tag: `v1.0`
+
+The repository store layout is represented as:
+
+```bash
+<root>
+└── v2
+    └── repositories
+        └── nginx
+            └── _manifests
+                └── revisions
+                    └── sha256
+                        └── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m
+                            └── link
+```
+
+Push a signature as blob and an ORAS Artifact that contains a blobs property referencing the signature, with the
+following properties:
+
+- digest: `sha256:222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i`
+- `subjectManifest` digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m`
+- `artifactType`: `application/vnd.example.artifact`
+
+On `PUT`, the artifact appears as a manifest revision. Additionally, an index entry is created under
+the subject to facilitate a lookup to the referrer. The index path where the entry is added is
+`/ref/<artifactType>`, as shown below.
+
+```
+<root>
+└── v2
+    └── repositories
+        └── nginx
+            └── _manifests
+                └── revisions
+                    └── sha256
+                        ├── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m
+                        │   ├── link
+                        │   └── ref
+                        │       └── digest(application/vnd.example.artifact)
+                        │           └── sha256
+                        │               └── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i
+                        │                   └── link
+                        └── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i
+                            └── link
+```
+
+Push another ORAS artifact with the following properties:
+
+- digest: `sha256:333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i`
+- `subjectManifest` digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m`
+- `artifactType`: `application/vnd.another.example.artifact`
+
+This results in an addition to the index as shown below.
+
+```
+<root>
+└── v2
+    └── repositories
+        └── nginx
+            └── _manifests
+                └── revisions
+                    └── sha256
+                        ├── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m
+                        │   ├── link
+                        │   └── ref
+                        │       ├── digest(application/vnd.example.artifact)
+                        │       │   └── sha256
+                        │       │       └── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i
+                        │       │           └── link
+                        │       └── digest(application/vnd.another.example.artifact)
+                        │           └── sha256
+                        │               └── 333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i
+                        │                   └── link
+                        ├── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i
+                        │   └── link
+                        └── 333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i
+                            └── link
+```
diff --git a/go.mod b/go.mod
@@ -2,6 +2,8 @@ module github.com/distribution/distribution/v3
 
 go 1.15
 
+replace github.com/oras-project/artifacts-spec => github.com/aviral26/artifacts-spec v0.0.2
+
 require (
 	github.com/Azure/azure-sdk-for-go v16.2.1+incompatible
 	github.com/Azure/go-autorest v10.8.1+incompatible // indirect
@@ -30,6 +32,7 @@ require (
 	github.com/ncw/swift v1.0.47
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/oras-project/artifacts-spec v0.0.0-00010101000000-000000000000
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v0.0.3

diff --git a/go.sum b/go.sum
@@ -8,6 +8,8 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/O
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aviral26/artifacts-spec v0.0.2 h1:uU5MIRT68TP4l2Elri9WoweDc9f8iSwe0G0ghDCxolQ=
+github.com/aviral26/artifacts-spec v0.0.2/go.mod h1:Xch2aLzSwtkhbFFN6LUzTfLtukYvMMdXJ4oZ8O7BOdc=
 github.com/aws/aws-sdk-go v1.34.9 h1:cUGBW9CVdi0mS7K1hDzxIqTpfeWhpoQiguq81M1tjK0=
 github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

diff --git a/manifest/orasartifact/manifest.go b/manifest/orasartifact/manifest.go
@@ -0,0 +1,111 @@
+package orasartifact
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/distribution/distribution/v3"
+	"github.com/opencontainers/go-digest"
+	v1 "github.com/oras-project/artifacts-spec/specs-go/v1"
+)
+
+func init() {
+	unmarshalFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		d := new(DeserializedManifest)
+		err := d.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		if d.inner.MediaType != v1.MediaTypeArtifactManifest {
+			err = fmt.Errorf("if present, mediaType in ORAS artifact manifest should be '%s' not '%s'",
+				v1.MediaTypeArtifactManifest, d.inner.MediaType)
+
+			return nil, distribution.Descriptor{}, err
+		}
+
+		dgst := digest.FromBytes(b)
+		return d, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeArtifactManifest}, err
+	}
+	err := distribution.RegisterManifestSchema(v1.MediaTypeArtifactManifest, unmarshalFunc)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register ORAS artifact manifest: %s", err))
+	}
+}
+
+// Manifest describes ORAS artifact manifests.
+type Manifest struct {
+	inner v1.Manifest
+}
+
+// ArtifactType returns the artifactType of this ORAS artifact.
+func (a Manifest) ArtifactType() string {
+	return a.inner.ArtifactType
+}
+
+// References returns the distribution descriptors for the referenced blobs.
+func (a Manifest) References() []distribution.Descriptor {
+	blobs := make([]distribution.Descriptor, len(a.inner.Blobs))
+	for i := range a.inner.Blobs {
+		blobs[i] = distribution.Descriptor{
+			MediaType: a.inner.Blobs[i].MediaType,
+			Digest:    a.inner.Blobs[i].Digest,
+			Size:      a.inner.Blobs[i].Size,
+		}
+	}
+	return blobs
+}
+
+// SubjectManifest returns the the subject manifest this artifact references.
+func (a Manifest) SubjectManifest() distribution.Descriptor {
+	return distribution.Descriptor{
+		MediaType: a.inner.SubjectManifest.MediaType,
+		Digest:    a.inner.SubjectManifest.Digest,
+		Size:      a.inner.SubjectManifest.Size,
+	}
+}
+
+// DeserializedManifest wraps Manifest with a copy of the original JSON data.
+type DeserializedManifest struct {
+	Manifest
+
+	// raw is the raw byte representation of the ORAS artifact.
+	raw []byte
+}
+
+// UnmarshalJSON populates a new Manifest struct from JSON data.
+func (d *DeserializedManifest) UnmarshalJSON(b []byte) error {
+	d.raw = make([]byte, len(b))
+	copy(d.raw, b)
+
+	var man v1.Manifest
+	if err := json.Unmarshal(d.raw, &man); err != nil {
+		return err
+	}
+	d.inner = man
+
+	return nil
+}
+
+// MarshalJSON returns the raw content.
+func (d *DeserializedManifest) MarshalJSON() ([]byte, error) {
+	if len(d.raw) > 0 {
+		return d.raw, nil
+	}
+
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
+}
+
+// Payload returns the raw content of the Artifact. The contents can be
+// used to calculate the content identifier.
+func (d DeserializedManifest) Payload() (string, []byte, error) {
+	var mediaType string
+	if d.inner.MediaType == "" {
+		mediaType = v1.MediaTypeArtifactManifest
+	} else {
+		mediaType = d.inner.MediaType
+	}
+
+	return mediaType, d.raw, nil
+}