From eedc4db0ab544dc70c52999b076fec010d83513c Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Tue, 2 May 2023 05:42:57 +0100 Subject: [PATCH 01/77] Added IRepository Signed-off-by: Samson Amaugo --- Oras.Tests/ContentTest/ContentTest.cs | 10 +++----- Oras.Tests/CopyTest.cs | 5 ++-- Oras/Interfaces/IBlobTarget.cs | 9 ++++++++ Oras/Interfaces/IDeleter.cs | 20 ++++++++++++++++ Oras/Interfaces/IFetcher.cs | 3 +++ Oras/Interfaces/IManifestTarget.cs | 10 ++++++++ Oras/Interfaces/IReferenceFetcher.cs | 21 +++++++++++++++++ Oras/Interfaces/IReferencePusher.cs | 23 +++++++++++++++++++ Oras/Interfaces/IReferrerLister.cs | 16 +++++++++++++ Oras/Interfaces/IRepository.cs | 18 +++++++++++++++ Oras/Interfaces/ITag.cs | 21 +++++++++++++++++ Oras/Interfaces/ITagLister.cs | 33 +++++++++++++++++++++++++++ Oras/Interfaces/ITagResolver.cs | 16 ++----------- 13 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 Oras/Interfaces/IBlobTarget.cs create mode 100644 Oras/Interfaces/IDeleter.cs create mode 100644 Oras/Interfaces/IManifestTarget.cs create mode 100644 Oras/Interfaces/IReferenceFetcher.cs create mode 100644 Oras/Interfaces/IReferencePusher.cs create mode 100644 Oras/Interfaces/IReferrerLister.cs create mode 100644 Oras/Interfaces/IRepository.cs create mode 100644 Oras/Interfaces/ITag.cs create mode 100644 Oras/Interfaces/ITagLister.cs diff --git a/Oras.Tests/ContentTest/ContentTest.cs b/Oras.Tests/ContentTest/ContentTest.cs index bcca311..f5698b6 100644 --- a/Oras.Tests/ContentTest/ContentTest.cs +++ b/Oras.Tests/ContentTest/ContentTest.cs @@ -1,10 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using static Oras.Content.Content; -using System.Text; -using System.Threading.Tasks; +using System.Text; using Xunit; +using static Oras.Content.Content; namespace Oras.Tests.ContentTest { @@ -19,7 +15,7 @@ public void CalculateDigest_VerifiesIfDigestMatches() var helloWorldDigest = "sha256:11d4ddc357e0822968dbfd226b6e1c2aac018d076a54da4f65e1dc8180684ac3"; var content = Encoding.UTF8.GetBytes("helloWorld"); var calculateHelloWorldDigest = CalculateDigest(content); - Assert.Equal(helloWorldDigest,calculateHelloWorldDigest); + Assert.Equal(helloWorldDigest, calculateHelloWorldDigest); } } } diff --git a/Oras.Tests/CopyTest.cs b/Oras.Tests/CopyTest.cs index dff3f5b..b1ae062 100644 --- a/Oras.Tests/CopyTest.cs +++ b/Oras.Tests/CopyTest.cs @@ -5,7 +5,6 @@ using System.Text.Json; using Xunit; using static Oras.Content.Content; -using Index = Oras.Models.Index; namespace Oras.Tests { @@ -63,7 +62,7 @@ public async Task Copy_CanCopyBetweenMemoryTargetsWithTaggedNode() var gotDesc = await Copy.CopyAsync(sourceTarget, reference, destinationTarget, "", cancellationToken); Assert.Equal(gotDesc, root); Assert.Equal(await destinationTarget.ResolveAsync(reference, cancellationToken), root); - + for (var i = 0; i < descs.Count; i++) { Assert.True(await destinationTarget.ExistsAsync(descs[i], cancellationToken)); @@ -121,7 +120,7 @@ public async Task CopyGraph_CanCopyBetweenMemoryTargets() var root = descs[3]; var destinationTarget = new MemoryTarget(); await Copy.CopyGraphAsync(sourceTarget, destinationTarget, root, cancellationToken); - for (var i=0; i + /// IBlobTarget is a CAS with the ability to stat and delete its content. + /// + internal interface IBlobTarget : ITarget, IResolver, IDeleter, IReferenceFetcher + { + } +} diff --git a/Oras/Interfaces/IDeleter.cs b/Oras/Interfaces/IDeleter.cs new file mode 100644 index 0000000..399f4b0 --- /dev/null +++ b/Oras/Interfaces/IDeleter.cs @@ -0,0 +1,20 @@ +using Oras.Models; +using System.Threading; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// IDeleter removes content. + /// + internal interface IDeleter + { + /// + /// This deletes content Identified by the descriptor + /// + /// + /// + /// + Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default); + } +} diff --git a/Oras/Interfaces/IFetcher.cs b/Oras/Interfaces/IFetcher.cs index 7422298..e63511c 100644 --- a/Oras/Interfaces/IFetcher.cs +++ b/Oras/Interfaces/IFetcher.cs @@ -5,6 +5,9 @@ namespace Oras.Interfaces { + /// + /// IFetcher fetches content. + /// public interface IFetcher { /// diff --git a/Oras/Interfaces/IManifestTarget.cs b/Oras/Interfaces/IManifestTarget.cs new file mode 100644 index 0000000..79fb44b --- /dev/null +++ b/Oras/Interfaces/IManifestTarget.cs @@ -0,0 +1,10 @@ +namespace Oras.Interfaces +{ + /// + /// IManifestTarget is a CAS with the ability to stat and delete its content. + /// Besides, ManifestTarget provides reference tagging. + /// + internal interface IManifestTarget : IBlobTarget, IReferencePusher, ITag + { + } +} diff --git a/Oras/Interfaces/IReferenceFetcher.cs b/Oras/Interfaces/IReferenceFetcher.cs new file mode 100644 index 0000000..81d71fb --- /dev/null +++ b/Oras/Interfaces/IReferenceFetcher.cs @@ -0,0 +1,21 @@ +using Oras.Models; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// IReferenceFetcher provides advanced fetch with the tag service. + /// + internal interface IReferenceFetcher + { + /// + /// FetchReferenceAsync fetches the content identified by the reference. + /// + /// + /// + /// + (Task, Task) FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); + } +} diff --git a/Oras/Interfaces/IReferencePusher.cs b/Oras/Interfaces/IReferencePusher.cs new file mode 100644 index 0000000..e0c05af --- /dev/null +++ b/Oras/Interfaces/IReferencePusher.cs @@ -0,0 +1,23 @@ +using Oras.Models; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// IReferencePusher provides advanced push with the tag service. + /// + internal interface IReferencePusher + { + /// + /// PushReferenceAsync pushes the manifest with a reference tag. + /// + /// + /// + /// + /// + /// + Task PushReferenceAsync(string reference, Descriptor descriptor, Stream content, CancellationToken cancellationToken = default); + } +} diff --git a/Oras/Interfaces/IReferrerLister.cs b/Oras/Interfaces/IReferrerLister.cs new file mode 100644 index 0000000..210ce0f --- /dev/null +++ b/Oras/Interfaces/IReferrerLister.cs @@ -0,0 +1,16 @@ +using Oras.Models; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// IReferrerLister provides the Referrers API. + /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers + /// + internal interface IReferrerLister + { + Task ReferrersAsync(Descriptor desc, string artifactType, Func fn, CancellationToken cancellationToken = default); + } +} diff --git a/Oras/Interfaces/IRepository.cs b/Oras/Interfaces/IRepository.cs new file mode 100644 index 0000000..c385713 --- /dev/null +++ b/Oras/Interfaces/IRepository.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + internal interface IRepository : ITarget, ITagResolver, IReferenceFetcher, IReferencePusher, IReferrerLister, IDeleter + { + /// + /// BlobsAsync provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. + /// + /// + Task BlobsAsync(); + /// + /// ManifestsAsync provides access to the manifest CAS only. + /// + /// + Task ManifestsAsync(); + } +} diff --git a/Oras/Interfaces/ITag.cs b/Oras/Interfaces/ITag.cs new file mode 100644 index 0000000..405391b --- /dev/null +++ b/Oras/Interfaces/ITag.cs @@ -0,0 +1,21 @@ +using Oras.Models; +using System.Threading; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// ITag tags reference tags + /// + public interface ITag + { + /// + /// TagAsync tags the descriptor with the reference. + /// + /// + /// + /// + /// + Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default); + } +} diff --git a/Oras/Interfaces/ITagLister.cs b/Oras/Interfaces/ITagLister.cs new file mode 100644 index 0000000..1cd4b03 --- /dev/null +++ b/Oras/Interfaces/ITagLister.cs @@ -0,0 +1,33 @@ +using System; +using System.Threading.Tasks; + +namespace Oras.Interfaces +{ + /// + /// ITagLister lists tags by the tag service. + /// + internal interface ITagLister + { + /// + /// Tags lists the tags available in the repository. + /// Since the returned tag list may be paginated by the underlying + /// implementation, a function should be passed in to process the paginated + /// tag list. + /// `last` argument is the `last` parameter when invoking the tags API. + /// If `last` is NOT empty, the entries in the response start after the + /// tag specified by `last`. Otherwise, the response starts from the top + /// of the Tags list. + /// Note: When implemented by a remote registry, the tags API is called. + /// However, not all registries supports pagination or conforms the + /// specification. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#content-discovery + /// - https://docs.docker.com/registry/spec/api/#tags + /// See also `Tags()` in this package. + /// + /// + /// + /// + Task TagsAsync(string last, Func fn); + } +} diff --git a/Oras/Interfaces/ITagResolver.cs b/Oras/Interfaces/ITagResolver.cs index 9bb40c2..f412c76 100644 --- a/Oras/Interfaces/ITagResolver.cs +++ b/Oras/Interfaces/ITagResolver.cs @@ -1,21 +1,9 @@ -using Oras.Models; -using System.Threading; -using System.Threading.Tasks; - -namespace Oras.Interfaces +namespace Oras.Interfaces { /// /// ITagResolver provides reference tag indexing services. /// - public interface ITagResolver : IResolver + public interface ITagResolver : IResolver, ITag { - /// - /// TagAsync tags the descriptor with the reference. - /// - /// - /// - /// - /// - Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default); } } From 1f78ac7706d2386b23280a4cfa551c985c8eef6f Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Tue, 2 May 2023 06:02:42 +0100 Subject: [PATCH 02/77] Added IRepository Signed-off-by: Samson Amaugo --- Oras/Interfaces/IRepository.cs | 12 ++++++++++++ Oras/Memory/MemoryStorage.cs | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Oras/Interfaces/IRepository.cs b/Oras/Interfaces/IRepository.cs index c385713..f5d1bfc 100644 --- a/Oras/Interfaces/IRepository.cs +++ b/Oras/Interfaces/IRepository.cs @@ -2,6 +2,18 @@ namespace Oras.Interfaces { + /// + /// Repository is an ORAS target and an union of the blob and the manifest CASs. + /// As specified by https://docs.docker.com/registry/spec/api/, it is natural to + /// assume that content.Resolver interface only works for manifests. Tagging a + /// blob may be resulted in an `ErrUnsupported` error. However, this interface + /// does not restrict tagging blobs. + /// Since a repository is an union of the blob and the manifest CASs, all + /// operations defined in the `BlobStore` are executed depending on the media + /// type of the given descriptor accordingly. + /// Furthermore, this interface also provides the ability to enforce the + /// separation of the blob and the manifests CASs. + /// internal interface IRepository : ITarget, ITagResolver, IReferenceFetcher, IReferencePusher, IReferrerLister, IDeleter { /// diff --git a/Oras/Memory/MemoryStorage.cs b/Oras/Memory/MemoryStorage.cs index 86ca640..b6a5332 100644 --- a/Oras/Memory/MemoryStorage.cs +++ b/Oras/Memory/MemoryStorage.cs @@ -44,7 +44,6 @@ public async Task PushAsync(Descriptor expected, Stream contentStream, Cancellat var added = _content.TryAdd(key, readBytes); if (!added) throw new AlreadyExistsException($"{key.Digest} : {key.MediaType}"); - return; } } From 4aa30f3c0ac1c9682d88a1e5f575e8dc73263e64 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 4 May 2023 08:38:23 +0100 Subject: [PATCH 03/77] made some changes Signed-off-by: Samson Amaugo --- Oras/Interfaces/IBlobTarget.cs | 9 --------- Oras/Interfaces/IManifestTarget.cs | 10 ---------- Oras/Interfaces/IReferrerLister.cs | 16 ---------------- Oras/Interfaces/Registry/IBlobStore.cs | 9 +++++++++ Oras/Interfaces/{ => Registry}/IDeleter.cs | 4 ++-- Oras/Interfaces/Registry/IManifestStore.cs | 10 ++++++++++ .../{ => Registry}/IReferenceFetcher.cs | 4 ++-- .../{ => Registry}/IReferencePusher.cs | 8 ++++---- Oras/Interfaces/{ => Registry}/IRepository.cs | 8 ++++---- Oras/Interfaces/{ => Registry}/ITagLister.cs | 12 +++++++----- 10 files changed, 38 insertions(+), 52 deletions(-) delete mode 100644 Oras/Interfaces/IBlobTarget.cs delete mode 100644 Oras/Interfaces/IManifestTarget.cs delete mode 100644 Oras/Interfaces/IReferrerLister.cs create mode 100644 Oras/Interfaces/Registry/IBlobStore.cs rename Oras/Interfaces/{ => Registry}/IDeleter.cs (88%) create mode 100644 Oras/Interfaces/Registry/IManifestStore.cs rename Oras/Interfaces/{ => Registry}/IReferenceFetcher.cs (89%) rename Oras/Interfaces/{ => Registry}/IReferencePusher.cs (73%) rename Oras/Interfaces/{ => Registry}/IRepository.cs (82%) rename Oras/Interfaces/{ => Registry}/ITagLister.cs (81%) diff --git a/Oras/Interfaces/IBlobTarget.cs b/Oras/Interfaces/IBlobTarget.cs deleted file mode 100644 index 739f22d..0000000 --- a/Oras/Interfaces/IBlobTarget.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Oras.Interfaces -{ - /// - /// IBlobTarget is a CAS with the ability to stat and delete its content. - /// - internal interface IBlobTarget : ITarget, IResolver, IDeleter, IReferenceFetcher - { - } -} diff --git a/Oras/Interfaces/IManifestTarget.cs b/Oras/Interfaces/IManifestTarget.cs deleted file mode 100644 index 79fb44b..0000000 --- a/Oras/Interfaces/IManifestTarget.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Oras.Interfaces -{ - /// - /// IManifestTarget is a CAS with the ability to stat and delete its content. - /// Besides, ManifestTarget provides reference tagging. - /// - internal interface IManifestTarget : IBlobTarget, IReferencePusher, ITag - { - } -} diff --git a/Oras/Interfaces/IReferrerLister.cs b/Oras/Interfaces/IReferrerLister.cs deleted file mode 100644 index 210ce0f..0000000 --- a/Oras/Interfaces/IReferrerLister.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Oras.Models; -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace Oras.Interfaces -{ - /// - /// IReferrerLister provides the Referrers API. - /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers - /// - internal interface IReferrerLister - { - Task ReferrersAsync(Descriptor desc, string artifactType, Func fn, CancellationToken cancellationToken = default); - } -} diff --git a/Oras/Interfaces/Registry/IBlobStore.cs b/Oras/Interfaces/Registry/IBlobStore.cs new file mode 100644 index 0000000..ed93828 --- /dev/null +++ b/Oras/Interfaces/Registry/IBlobStore.cs @@ -0,0 +1,9 @@ +namespace Oras.Interfaces.Registry +{ + /// + /// IBlobStore is a CAS with the ability to stat and delete its content. + /// + public interface IBlobStore : IResolver, IDeleter, IReferenceFetcher + { + } +} diff --git a/Oras/Interfaces/IDeleter.cs b/Oras/Interfaces/Registry/IDeleter.cs similarity index 88% rename from Oras/Interfaces/IDeleter.cs rename to Oras/Interfaces/Registry/IDeleter.cs index 399f4b0..050eedf 100644 --- a/Oras/Interfaces/IDeleter.cs +++ b/Oras/Interfaces/Registry/IDeleter.cs @@ -2,12 +2,12 @@ using System.Threading; using System.Threading.Tasks; -namespace Oras.Interfaces +namespace Oras.Interfaces.Registry { /// /// IDeleter removes content. /// - internal interface IDeleter + public interface IDeleter { /// /// This deletes content Identified by the descriptor diff --git a/Oras/Interfaces/Registry/IManifestStore.cs b/Oras/Interfaces/Registry/IManifestStore.cs new file mode 100644 index 0000000..01123d0 --- /dev/null +++ b/Oras/Interfaces/Registry/IManifestStore.cs @@ -0,0 +1,10 @@ +namespace Oras.Interfaces.Registry +{ + /// + /// IManifestStore is a CAS with the ability to stat and delete its content. + /// Besides, ManifestStore provides reference tagging. + /// + public interface IManifestStore : IBlobStore, IReferencePusher, ITag + { + } +} diff --git a/Oras/Interfaces/IReferenceFetcher.cs b/Oras/Interfaces/Registry/IReferenceFetcher.cs similarity index 89% rename from Oras/Interfaces/IReferenceFetcher.cs rename to Oras/Interfaces/Registry/IReferenceFetcher.cs index 81d71fb..3d8ce6a 100644 --- a/Oras/Interfaces/IReferenceFetcher.cs +++ b/Oras/Interfaces/Registry/IReferenceFetcher.cs @@ -3,12 +3,12 @@ using System.Threading; using System.Threading.Tasks; -namespace Oras.Interfaces +namespace Oras.Interfaces.Registry { /// /// IReferenceFetcher provides advanced fetch with the tag service. /// - internal interface IReferenceFetcher + public interface IReferenceFetcher { /// /// FetchReferenceAsync fetches the content identified by the reference. diff --git a/Oras/Interfaces/IReferencePusher.cs b/Oras/Interfaces/Registry/IReferencePusher.cs similarity index 73% rename from Oras/Interfaces/IReferencePusher.cs rename to Oras/Interfaces/Registry/IReferencePusher.cs index e0c05af..cb21b0a 100644 --- a/Oras/Interfaces/IReferencePusher.cs +++ b/Oras/Interfaces/Registry/IReferencePusher.cs @@ -3,21 +3,21 @@ using System.Threading; using System.Threading.Tasks; -namespace Oras.Interfaces +namespace Oras.Interfaces.Registry { /// /// IReferencePusher provides advanced push with the tag service. /// - internal interface IReferencePusher + public interface IReferencePusher { /// /// PushReferenceAsync pushes the manifest with a reference tag. /// - /// /// /// + /// /// /// - Task PushReferenceAsync(string reference, Descriptor descriptor, Stream content, CancellationToken cancellationToken = default); + Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, CancellationToken cancellationToken = default); } } diff --git a/Oras/Interfaces/IRepository.cs b/Oras/Interfaces/Registry/IRepository.cs similarity index 82% rename from Oras/Interfaces/IRepository.cs rename to Oras/Interfaces/Registry/IRepository.cs index f5d1bfc..6f526b8 100644 --- a/Oras/Interfaces/IRepository.cs +++ b/Oras/Interfaces/Registry/IRepository.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace Oras.Interfaces +namespace Oras.Interfaces.Registry { /// /// Repository is an ORAS target and an union of the blob and the manifest CASs. @@ -14,17 +14,17 @@ namespace Oras.Interfaces /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// - internal interface IRepository : ITarget, ITagResolver, IReferenceFetcher, IReferencePusher, IReferrerLister, IDeleter + internal interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeleter { /// /// BlobsAsync provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. /// /// - Task BlobsAsync(); + Task BlobsAsync(); /// /// ManifestsAsync provides access to the manifest CAS only. /// /// - Task ManifestsAsync(); + Task ManifestsAsync(); } } diff --git a/Oras/Interfaces/ITagLister.cs b/Oras/Interfaces/Registry/ITagLister.cs similarity index 81% rename from Oras/Interfaces/ITagLister.cs rename to Oras/Interfaces/Registry/ITagLister.cs index 1cd4b03..26e9994 100644 --- a/Oras/Interfaces/ITagLister.cs +++ b/Oras/Interfaces/Registry/ITagLister.cs @@ -1,12 +1,13 @@ using System; +using System.Threading; using System.Threading.Tasks; -namespace Oras.Interfaces +namespace Oras.Interfaces.Registry { /// /// ITagLister lists tags by the tag service. /// - internal interface ITagLister + public interface ITagLister { /// /// Tags lists the tags available in the repository. @@ -21,13 +22,14 @@ internal interface ITagLister /// However, not all registries supports pagination or conforms the /// specification. /// References: - /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#content-discovery + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md /// - https://docs.docker.com/registry/spec/api/#tags /// See also `Tags()` in this package. /// /// /// + /// /// - Task TagsAsync(string last, Func fn); + Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default); } -} +} \ No newline at end of file From bbd7b1e2b17e28b9f51e492aa3cfab665a63deac Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 4 May 2023 08:50:45 +0100 Subject: [PATCH 04/77] made some changes Signed-off-by: Samson Amaugo --- Oras/Interfaces/ITagResolver.cs | 2 +- Oras/Interfaces/{ITag.cs => ITagger.cs} | 4 ++-- Oras/Interfaces/Registry/IManifestStore.cs | 2 +- Oras/Interfaces/Registry/IRepository.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename Oras/Interfaces/{ITag.cs => ITagger.cs} (89%) diff --git a/Oras/Interfaces/ITagResolver.cs b/Oras/Interfaces/ITagResolver.cs index f412c76..cae2cfd 100644 --- a/Oras/Interfaces/ITagResolver.cs +++ b/Oras/Interfaces/ITagResolver.cs @@ -3,7 +3,7 @@ /// /// ITagResolver provides reference tag indexing services. /// - public interface ITagResolver : IResolver, ITag + public interface ITagResolver : IResolver, ITagger { } } diff --git a/Oras/Interfaces/ITag.cs b/Oras/Interfaces/ITagger.cs similarity index 89% rename from Oras/Interfaces/ITag.cs rename to Oras/Interfaces/ITagger.cs index 405391b..c3146a0 100644 --- a/Oras/Interfaces/ITag.cs +++ b/Oras/Interfaces/ITagger.cs @@ -5,9 +5,9 @@ namespace Oras.Interfaces { /// - /// ITag tags reference tags + /// ITagger tags reference tags /// - public interface ITag + public interface ITagger { /// /// TagAsync tags the descriptor with the reference. diff --git a/Oras/Interfaces/Registry/IManifestStore.cs b/Oras/Interfaces/Registry/IManifestStore.cs index 01123d0..d737907 100644 --- a/Oras/Interfaces/Registry/IManifestStore.cs +++ b/Oras/Interfaces/Registry/IManifestStore.cs @@ -4,7 +4,7 @@ /// IManifestStore is a CAS with the ability to stat and delete its content. /// Besides, ManifestStore provides reference tagging. /// - public interface IManifestStore : IBlobStore, IReferencePusher, ITag + public interface IManifestStore : IBlobStore, IReferencePusher, ITagger { } } diff --git a/Oras/Interfaces/Registry/IRepository.cs b/Oras/Interfaces/Registry/IRepository.cs index 6f526b8..c079f54 100644 --- a/Oras/Interfaces/Registry/IRepository.cs +++ b/Oras/Interfaces/Registry/IRepository.cs @@ -14,7 +14,7 @@ namespace Oras.Interfaces.Registry /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// - internal interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeleter + public interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeleter { /// /// BlobsAsync provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. From fefed02450d4c014e333893487d7a55fe8bdeea5 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 5 May 2023 17:53:51 +0100 Subject: [PATCH 05/77] made some changes Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IReferenceFetcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Interfaces/Registry/IReferenceFetcher.cs b/Oras/Interfaces/Registry/IReferenceFetcher.cs index 3d8ce6a..b053cc1 100644 --- a/Oras/Interfaces/Registry/IReferenceFetcher.cs +++ b/Oras/Interfaces/Registry/IReferenceFetcher.cs @@ -16,6 +16,6 @@ public interface IReferenceFetcher /// /// /// - (Task, Task) FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); + Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); } } From a1e139164fc7d6323fcdc98fc81e115bdffe3ffd Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 07:29:38 +0100 Subject: [PATCH 06/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IBlobStore.cs | 2 +- Oras/Interfaces/Registry/IRepository.cs | 14 ++++++-------- Oras/Interfaces/Registry/ITagLister.cs | 9 ++------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Oras/Interfaces/Registry/IBlobStore.cs b/Oras/Interfaces/Registry/IBlobStore.cs index ed93828..157af13 100644 --- a/Oras/Interfaces/Registry/IBlobStore.cs +++ b/Oras/Interfaces/Registry/IBlobStore.cs @@ -3,7 +3,7 @@ /// /// IBlobStore is a CAS with the ability to stat and delete its content. /// - public interface IBlobStore : IResolver, IDeleter, IReferenceFetcher + public interface IBlobStore : IResolver, IDeleter, IReferenceFetcher, IStorage { } } diff --git a/Oras/Interfaces/Registry/IRepository.cs b/Oras/Interfaces/Registry/IRepository.cs index c079f54..f557c05 100644 --- a/Oras/Interfaces/Registry/IRepository.cs +++ b/Oras/Interfaces/Registry/IRepository.cs @@ -1,6 +1,4 @@ -using System.Threading.Tasks; - -namespace Oras.Interfaces.Registry +namespace Oras.Interfaces.Registry { /// /// Repository is an ORAS target and an union of the blob and the manifest CASs. @@ -14,17 +12,17 @@ namespace Oras.Interfaces.Registry /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// - public interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeleter + public interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeleter, ITagLister { /// - /// BlobsAsync provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. + /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. /// /// - Task BlobsAsync(); + IBlobStore Blobs(); /// - /// ManifestsAsync provides access to the manifest CAS only. + /// Manifests provides access to the manifest CAS only. /// /// - Task ManifestsAsync(); + IManifestStore Manifests(); } } diff --git a/Oras/Interfaces/Registry/ITagLister.cs b/Oras/Interfaces/Registry/ITagLister.cs index 26e9994..c633e7d 100644 --- a/Oras/Interfaces/Registry/ITagLister.cs +++ b/Oras/Interfaces/Registry/ITagLister.cs @@ -14,20 +14,15 @@ public interface ITagLister /// Since the returned tag list may be paginated by the underlying /// implementation, a function should be passed in to process the paginated /// tag list. - /// `last` argument is the `last` parameter when invoking the tags API. - /// If `last` is NOT empty, the entries in the response start after the - /// tag specified by `last`. Otherwise, the response starts from the top - /// of the Tags list. /// Note: When implemented by a remote registry, the tags API is called. /// However, not all registries supports pagination or conforms the /// specification. /// References: /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md /// - https://docs.docker.com/registry/spec/api/#tags - /// See also `Tags()` in this package. /// - /// - /// + /// The `last` parameter when invoking the tags API. If `last` is NOT empty, the entries in the response start after the tag specified by `last`. Otherwise, the response starts from the top of the Tags list. + /// The function to process the paginated tag list /// /// Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default); From 454d8dd9cf6f618fc63eae1d46ed062f56a8798c Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 07:30:25 +0100 Subject: [PATCH 07/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/ITagLister.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Interfaces/Registry/ITagLister.cs b/Oras/Interfaces/Registry/ITagLister.cs index c633e7d..6c41b0d 100644 --- a/Oras/Interfaces/Registry/ITagLister.cs +++ b/Oras/Interfaces/Registry/ITagLister.cs @@ -27,4 +27,4 @@ public interface ITagLister /// Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default); } -} \ No newline at end of file +} From 127aa30681263ccb7b80278db547b320eabc7217 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 07:31:05 +0100 Subject: [PATCH 08/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IBlobStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Interfaces/Registry/IBlobStore.cs b/Oras/Interfaces/Registry/IBlobStore.cs index 157af13..cedada4 100644 --- a/Oras/Interfaces/Registry/IBlobStore.cs +++ b/Oras/Interfaces/Registry/IBlobStore.cs @@ -3,7 +3,7 @@ /// /// IBlobStore is a CAS with the ability to stat and delete its content. /// - public interface IBlobStore : IResolver, IDeleter, IReferenceFetcher, IStorage + public interface IBlobStore : IStorage, IResolver, IDeleter, IReferenceFetcher { } } From 7c1af7822b84d17e0468f5ae346c10a56af13f31 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 07:36:19 +0100 Subject: [PATCH 09/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/{Registry => }/IDeleter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Oras/Interfaces/{Registry => }/IDeleter.cs (93%) diff --git a/Oras/Interfaces/Registry/IDeleter.cs b/Oras/Interfaces/IDeleter.cs similarity index 93% rename from Oras/Interfaces/Registry/IDeleter.cs rename to Oras/Interfaces/IDeleter.cs index 050eedf..8c46404 100644 --- a/Oras/Interfaces/Registry/IDeleter.cs +++ b/Oras/Interfaces/IDeleter.cs @@ -2,7 +2,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Oras.Interfaces.Registry +namespace Oras.Interfaces { /// /// IDeleter removes content. From 3ba1fd15f1deddb8fa45b7585210e0376d99aab4 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 09:10:31 +0100 Subject: [PATCH 10/77] Update Oras/Interfaces/Registry/IRepository.cs Co-authored-by: Lixia (Sylvia) Lei Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IRepository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Interfaces/Registry/IRepository.cs b/Oras/Interfaces/Registry/IRepository.cs index f557c05..a6cc341 100644 --- a/Oras/Interfaces/Registry/IRepository.cs +++ b/Oras/Interfaces/Registry/IRepository.cs @@ -3,7 +3,7 @@ /// /// Repository is an ORAS target and an union of the blob and the manifest CASs. /// As specified by https://docs.docker.com/registry/spec/api/, it is natural to - /// assume that content.Resolver interface only works for manifests. Tagging a + /// assume that IResolver interface only works for manifests. Tagging a /// blob may be resulted in an `ErrUnsupported` error. However, this interface /// does not restrict tagging blobs. /// Since a repository is an union of the blob and the manifest CASs, all From 8efb2cbd7758f1a22e78b310d7dcae8f97be5871 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 09:11:49 +0100 Subject: [PATCH 11/77] created registry folder Signed-off-by: Samson Amaugo --- Oras/Oras.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index 0876bf0..c63e5ab 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -11,4 +11,7 @@ + + + From 12f516253c5287736d876377a6fb7b5ec3fbedc7 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 09:15:32 +0100 Subject: [PATCH 12/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/ITagLister.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Interfaces/Registry/ITagLister.cs b/Oras/Interfaces/Registry/ITagLister.cs index 6c41b0d..a273127 100644 --- a/Oras/Interfaces/Registry/ITagLister.cs +++ b/Oras/Interfaces/Registry/ITagLister.cs @@ -10,7 +10,7 @@ namespace Oras.Interfaces.Registry public interface ITagLister { /// - /// Tags lists the tags available in the repository. + /// TagsAsync lists the tags available in the repository. /// Since the returned tag list may be paginated by the underlying /// implementation, a function should be passed in to process the paginated /// tag list. From 6ccbc6089962723cb5d02517f22ff2fce4b147cc Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 09:16:16 +0100 Subject: [PATCH 13/77] made some changes to some interfaces Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IManifestStore.cs | 2 +- Oras/Interfaces/Registry/IRepository.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Oras/Interfaces/Registry/IManifestStore.cs b/Oras/Interfaces/Registry/IManifestStore.cs index d737907..5d73b6b 100644 --- a/Oras/Interfaces/Registry/IManifestStore.cs +++ b/Oras/Interfaces/Registry/IManifestStore.cs @@ -2,7 +2,7 @@ { /// /// IManifestStore is a CAS with the ability to stat and delete its content. - /// Besides, ManifestStore provides reference tagging. + /// Besides, IManifestStore provides reference tagging. /// public interface IManifestStore : IBlobStore, IReferencePusher, ITagger { diff --git a/Oras/Interfaces/Registry/IRepository.cs b/Oras/Interfaces/Registry/IRepository.cs index a6cc341..9edeed4 100644 --- a/Oras/Interfaces/Registry/IRepository.cs +++ b/Oras/Interfaces/Registry/IRepository.cs @@ -4,10 +4,10 @@ /// Repository is an ORAS target and an union of the blob and the manifest CASs. /// As specified by https://docs.docker.com/registry/spec/api/, it is natural to /// assume that IResolver interface only works for manifests. Tagging a - /// blob may be resulted in an `ErrUnsupported` error. However, this interface + /// blob may be resulted in an `UnsupportedException` error. However, this interface /// does not restrict tagging blobs. /// Since a repository is an union of the blob and the manifest CASs, all - /// operations defined in the `BlobStore` are executed depending on the media + /// operations defined in the `IBlobStore` are executed depending on the media /// type of the given descriptor accordingly. /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. From 3779cc963ec61dbe1fb3486db9e0926195bb3e6d Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 8 May 2023 12:13:30 +0100 Subject: [PATCH 14/77] adding extra features Signed-off-by: Samson Amaugo --- Oras/Oras.csproj | 3 -- Oras/Remote/Reference.cs | 74 +++++++++++++++++++++++++++++++++++++++ Oras/Remote/Registry.cs | 9 +++++ Oras/Remote/Repository.cs | 59 +++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 Oras/Remote/Reference.cs create mode 100644 Oras/Remote/Registry.cs create mode 100644 Oras/Remote/Repository.cs diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index c63e5ab..0876bf0 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -11,7 +11,4 @@ - - - diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs new file mode 100644 index 0000000..07f4de0 --- /dev/null +++ b/Oras/Remote/Reference.cs @@ -0,0 +1,74 @@ +using Oras.Exceptions; + +namespace Oras.Remote +{ + public class ReferenceObj + { + /// + /// Registry is the name of the registry. It is usually the domain name of the registry optionally with a port. + /// + public string Registry { get; set; } + + /// + /// Repository is the name of the repository. + /// + public string Repository { get; set; } + + /// + /// Ref is the reference of the object in the repository. This field + /// can take any one of the four valid forms (see ParseReference). In the + /// case where it's the empty string, it necessarily implies valid form D, + /// and where it is non-empty, then it is either a tag, or a digest + /// (implying one of valid forms A, B, or C). + /// + public string Reference { get; set; } + + /// + /// repositoryRegexp is adapted from the distribution implementation. The + /// repository name set under OCI distribution spec is a subset of the docker + /// repositoryRegexp is adapted from the distribution implementation. The + /// spec. For maximum compatability, the docker spec is verified client-side. + /// Further checks are left to the server-side. + /// References: + /// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53 + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#pulling-manifests + /// + const string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; + + /// + /// tagRegexp checks the tag name. + /// The docker and OCI spec have the same regular expression. + /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#pulling-manifests + /// + const string tagRegexp = @"^[\w][\w.-]{0,127}$"; + + public ReferenceObj ParseReference(string artifact) + { + var parts = artifact.Split("/", 2); + if (parts.Length == 1) + { + throw new InvalidReferenceException($"missing repository"); + } + (var registry, var path) = (parts[0], parts[1]); + bool isTag; + string repository; + string reference; + var index = path.IndexOf("@"); + if (index != -1) + { + // digest found; Valid From A (if not B) + isTag = false; + repository = path.Substring(0, index); + reference = path.Substring(index + 1); + + if (repository.IndexOf(":") is var colonIndex && colonIndex != -1) + { + // Valid From B + throw new InvalidReferenceException($"invalid reference format: {artifact}"); + } + } + + return default; + } + } +} diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs new file mode 100644 index 0000000..54f0b3a --- /dev/null +++ b/Oras/Remote/Registry.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace Oras.Remote +{ + public class Registry + { + public async Task Ping() + } +} diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs new file mode 100644 index 0000000..5d035ea --- /dev/null +++ b/Oras/Remote/Repository.cs @@ -0,0 +1,59 @@ +namespace Oras.Remote +{ + /// + /// Repository is an HTTP client to a remote repository + /// + public class Repository + { + + // Client is the underlying HTTP client used to access the remote registry. + // If nil, auth.DefaultClient is used. + public + Client Client + + // Reference references the remote repository. + Reference registry.Reference + + // PlainHTTP signals the transport to access the remote repository via HTTP + // instead of HTTPS. + PlainHTTP bool + + // ManifestMediaTypes is used in `Accept` header for resolving manifests + // from references. It is also used in identifying manifests and blobs from + // descriptors. If an empty list is present, default manifest media types + // are used. + ManifestMediaTypes []string + + // TagListPageSize specifies the page size when invoking the tag list API. + // If zero, the page size is determined by the remote registry. + // Reference: https://docs.docker.com/registry/spec/api/#tags + TagListPageSize int + + // ReferrerListPageSize specifies the page size when invoking the Referrers + // API. + // If zero, the page size is determined by the remote registry. + // Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers + ReferrerListPageSize int + + // MaxMetadataBytes specifies a limit on how many response bytes are allowed + // in the server's response to the metadata APIs, such as catalog list, tag + // list, and referrers list. + // If less than or equal to zero, a default (currently 4MiB) is used. + MaxMetadataBytes int64 + + // NOTE: Must keep fields in sync with newRepositoryWithOptions function. + + // referrersState represents that if the repository supports Referrers API. + // default: referrersStateUnknown + referrersState referrersState + + // referrersPingLock locks the pingReferrers() method and allows only + // one go-routine to send the request. + referrersPingLock sync.Mutex + + // referrersMergePool provides a way to manage concurrent updates to a + // referrers index tagged by referrers tag schema. + referrersMergePool syncutil.Pool[syncutil.Merge[referrerChange]] +} +} +} From 42c61fe03762ef1130e5c559d304a40900233c2f Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Tue, 9 May 2023 13:38:05 +0100 Subject: [PATCH 15/77] =?UTF-8?q?adding=20extra=20features=20omo=20?= =?UTF-8?q?=F0=9F=98=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IRepositoryOption.cs | 52 +++ Oras/Remote/ManifestUtil.cs | 38 ++ Oras/Remote/Reference.cs | 120 ++++- Oras/Remote/Registry.cs | 6 +- Oras/Remote/RegistryUtil.cs | 25 + Oras/Remote/Repository.cs | 436 +++++++++++++++--- 6 files changed, 612 insertions(+), 65 deletions(-) create mode 100644 Oras/Interfaces/Registry/IRepositoryOption.cs create mode 100644 Oras/Remote/ManifestUtil.cs create mode 100644 Oras/Remote/RegistryUtil.cs diff --git a/Oras/Interfaces/Registry/IRepositoryOption.cs b/Oras/Interfaces/Registry/IRepositoryOption.cs new file mode 100644 index 0000000..f1f96d7 --- /dev/null +++ b/Oras/Interfaces/Registry/IRepositoryOption.cs @@ -0,0 +1,52 @@ +using Oras.Remote; +using System.Net.Http; + +namespace Oras.Interfaces.Registry +{ + /// + /// IRepositoryOption is used to configure a remote repository. + /// + public interface IRepositoryOption + { + /// + /// Client is the underlying HTTP client used to access the remote registry. + /// + public HttpClient Client { get; set; } + + /// + /// Reference references the remote repository. + /// + public ReferenceObj Reference { get; set; } + + /// + /// PlainHTTP signals the transport to access the remote repository via HTTP + /// instead of HTTPS. + /// + public bool PlainHTTP { get; set; } + + + /// + /// ManifestMediaTypes is used in `Accept` header for resolving manifests + /// from references. It is also used in identifying manifests and blobs from + /// descriptors. If an empty list is present, default manifest media types + /// are used. + /// + public string[] ManifestMediaTypes { get; set; } + + /// + /// TagListPageSize specifies the page size when invoking the tag list API. + /// If zero, the page size is determined by the remote registry. + /// Reference: https://docs.docker.com/registry/spec/api/#tags + /// + public int TagListPageSize { get; set; } + + /// + /// MaxMetadataBytes specifies a limit on how many response bytes are allowed + /// in the server's response to the metadata APIs, such as catalog list, tag + /// list, and referrers list. + /// If less than or equal to zero, a default (currently 4MiB) is used. + /// + public long MaxMetadataBytes { get; set; } + + } +} diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs new file mode 100644 index 0000000..aa4f69c --- /dev/null +++ b/Oras/Remote/ManifestUtil.cs @@ -0,0 +1,38 @@ +using Oras.Constants; +using Oras.Models; +using System.Linq; + +namespace Oras.Remote +{ + static class ManifestUtil + { + public static string[] DefaultManifestMediaTypes = new[] + { + DockerMediaTypes.Manifest, + DockerMediaTypes.ManifestList, + OCIMediaTypes.ImageIndex, + OCIMediaTypes.ImageManifest + }; + + /// + /// isManifest determines if the given descriptor is a manifest. + /// + /// + /// + /// + public static bool isManifest(string[] manifestMediaTypes, Descriptor desc) + { + if (manifestMediaTypes.Length == 0) + { + manifestMediaTypes = DefaultManifestMediaTypes; + } + + if (manifestMediaTypes.Any((mediaType) => mediaType == desc.MediaType)) + { + return true; + } + + return false; + } + } +} \ No newline at end of file diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 07f4de0..0498fd3 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -1,4 +1,6 @@ using Oras.Exceptions; +using System; +using System.Text.RegularExpressions; namespace Oras.Remote { @@ -42,6 +44,10 @@ public class ReferenceObj /// const string tagRegexp = @"^[\w][\w.-]{0,127}$"; + /// + /// digestRegexp checks the digest. + /// + const string digestRegexp = @"^sha256:[0-9a-fA-F]{64}$"; public ReferenceObj ParseReference(string artifact) { var parts = artifact.Split("/", 2); @@ -50,25 +56,121 @@ public ReferenceObj ParseReference(string artifact) throw new InvalidReferenceException($"missing repository"); } (var registry, var path) = (parts[0], parts[1]); - bool isTag; - string repository; - string reference; - var index = path.IndexOf("@"); - if (index != -1) + bool isTag = false; + string repository = String.Empty; + string reference = String.Empty; + + if (path.IndexOf("@") is var index && index != -1) { // digest found; Valid From A (if not B) isTag = false; repository = path.Substring(0, index); reference = path.Substring(index + 1); - if (repository.IndexOf(":") is var colonIndex && colonIndex != -1) + if (repository.IndexOf(":") is var indexOfColon && indexOfColon != -1) { - // Valid From B - throw new InvalidReferenceException($"invalid reference format: {artifact}"); + // tag found ( and now dropped without validation ) since the + // digest already present; Valid Form B + repository = path.Substring(0, indexOfColon); } } + else if (path.IndexOf(":") is var indexOfColon && indexOfColon != -1) + { + // tag found; Valid Form C + isTag = true; + repository = path.Substring(0, indexOfColon); + reference = path.Substring(indexOfColon + 1); + } + else + { + // empty `reference`; Valid Form D + repository = path; + } + var refObj = new ReferenceObj + { + Registry = registry, + Repository = repository, + Reference = reference + }; + ValidateRegistry(); + ValidateRepository(); + + if (Reference.Length == 0) + { + return refObj; + } + + ValidateReferenceAsDigest(); + if (isTag) + { + ValidateReferenceAsTag(); + } + return refObj; + } + + /// + /// ValidateReferenceAsDigest validates the reference as a digest. + /// + public void ValidateReferenceAsDigest() + { + - return default; + if (!Regex.IsMatch(Reference, digestRegexp)) + { + throw new InvalidReferenceException($"invalid reference format: {Reference}"); + } + } + + /// + /// ValidateRepository checks if the repository name is valid. + /// + /// + public void ValidateRepository() + { + if (!Regex.IsMatch(Reference, repositoryRegexp)) + { + throw new InvalidReferenceException("Invalid Respository"); + } + } + + /// + /// ValidateRegistry checks if the registry path is valid. + /// + /// + public void ValidateRegistry() + { + if (!Uri.IsWellFormedUriString("dummy://" + this.Registry, UriKind.Absolute)) + { + throw new InvalidReferenceException("Invalid Registry"); + }; + } + + public void ValidateReferenceAsTag() + { + if (!Regex.IsMatch(Reference, tagRegexp)) + { + throw new InvalidReferenceException("Invalid Tag"); + } + } + + /// + /// ValidateReference where the reference is first tried as an ampty string, then + /// as a digest, and if that fails, as a tag. + /// + public void ValidateReference() + { + if (Reference.Length == 0) + { + return; + } + if (Reference.IndexOf("@") != -1) + { + ValidateReferenceAsDigest(); + } + else + { + ValidateReferenceAsTag(); + } } } } diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 54f0b3a..f9d7628 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -1,9 +1,7 @@ -using System.Threading.Tasks; - -namespace Oras.Remote +namespace Oras.Remote { public class Registry { - public async Task Ping() + // public async Task Ping() } } diff --git a/Oras/Remote/RegistryUtil.cs b/Oras/Remote/RegistryUtil.cs new file mode 100644 index 0000000..25ab301 --- /dev/null +++ b/Oras/Remote/RegistryUtil.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Oras.Remote +{ + + internal static class RegistryUtil + { + /// + /// BuildScheme returns HTTP scheme used to access the remote registry. + /// + /// + /// + public static string BuildScheme(bool plainHTTP) + { + if (plainHTTP) + { + return "http"; + } + + return "https"; + } + } +} diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 5d035ea..ea564d4 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -1,59 +1,391 @@ -namespace Oras.Remote +using Oras.Interfaces.Registry; +using Oras.Models; +using System; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Oras.Exceptions; + +namespace Oras.Remote { + public class RepositoryOption : IRepositoryOption + { + public HttpClient Client { get; set; } + public ReferenceObj Reference { get; set; } + public bool PlainHTTP { get; set; } + public string[] ManifestMediaTypes { get; set; } + public int TagListPageSize { get; set; } + public long MaxMetadataBytes { get; set; } + } /// /// Repository is an HTTP client to a remote repository /// - public class Repository + public class Repository : IRepository, IRepositoryOption { + /// + /// Client is the underlying HTTP client used to access the remote registry. + /// + public HttpClient Client { get; set; } - // Client is the underlying HTTP client used to access the remote registry. - // If nil, auth.DefaultClient is used. - public - Client Client - - // Reference references the remote repository. - Reference registry.Reference - - // PlainHTTP signals the transport to access the remote repository via HTTP - // instead of HTTPS. - PlainHTTP bool - - // ManifestMediaTypes is used in `Accept` header for resolving manifests - // from references. It is also used in identifying manifests and blobs from - // descriptors. If an empty list is present, default manifest media types - // are used. - ManifestMediaTypes []string - - // TagListPageSize specifies the page size when invoking the tag list API. - // If zero, the page size is determined by the remote registry. - // Reference: https://docs.docker.com/registry/spec/api/#tags - TagListPageSize int - - // ReferrerListPageSize specifies the page size when invoking the Referrers - // API. - // If zero, the page size is determined by the remote registry. - // Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers - ReferrerListPageSize int - - // MaxMetadataBytes specifies a limit on how many response bytes are allowed - // in the server's response to the metadata APIs, such as catalog list, tag - // list, and referrers list. - // If less than or equal to zero, a default (currently 4MiB) is used. - MaxMetadataBytes int64 - - // NOTE: Must keep fields in sync with newRepositoryWithOptions function. - - // referrersState represents that if the repository supports Referrers API. - // default: referrersStateUnknown - referrersState referrersState - - // referrersPingLock locks the pingReferrers() method and allows only - // one go-routine to send the request. - referrersPingLock sync.Mutex - - // referrersMergePool provides a way to manage concurrent updates to a - // referrers index tagged by referrers tag schema. - referrersMergePool syncutil.Pool[syncutil.Merge[referrerChange]] -} -} + /// + /// ReferenceObj references the remote repository. + /// + public ReferenceObj Reference { get; set; } + + /// + /// PlainHTTP signals the transport to access the remote repository via HTTP + /// instead of HTTPS. + /// + public bool PlainHTTP { get; set; } + + + /// + /// ManifestMediaTypes is used in `Accept` header for resolving manifests + /// from references. It is also used in identifying manifests and blobs from + /// descriptors. If an empty list is present, default manifest media types + /// are used. + /// + public string[] ManifestMediaTypes { get; set; } + + /// + /// TagListPageSize specifies the page size when invoking the tag list API. + /// If zero, the page size is determined by the remote registry. + /// Reference: https://docs.docker.com/registry/spec/api/#tags + /// + public int TagListPageSize { get; set; } + + /// + /// MaxMetadataBytes specifies a limit on how many response bytes are allowed + /// in the server's response to the metadata APIs, such as catalog list, tag + /// list, and referrers list. + /// If less than or equal to zero, a default (currently 4MiB) is used. + /// + public long MaxMetadataBytes { get; set; } + + /// + /// Creates a client to the remote repository identified by a reference + /// Example: localhost:5000/hello-world + /// + /// + public Repository(string reference) + { + var refObj = new ReferenceObj().ParseReference(reference); + Reference = refObj; + } + + /// + /// This constructor customizes the Properties using the values + /// from the RepositoryOptions. + /// RepositoryOptions contains unexported state that must not be copied + /// to multiple Repositories. To handle this we explicitly copy only the + /// fields that we want to reproduce. + /// + /// + /// + public Repository(ReferenceObj refObj, RepositoryOption option) + { + refObj.ValidateRepository(); + Client = option.Client; + Reference = refObj; + PlainHTTP = option.PlainHTTP; + ManifestMediaTypes = option.ManifestMediaTypes; + TagListPageSize = option.TagListPageSize; + MaxMetadataBytes = option.MaxMetadataBytes; + + } + + /// + /// client returns an HTTP client used to access the remote repository. + /// A default HTTP client is return if the client is not configured. + /// + /// + private HttpClient client() + { + if (Client is null) + { + var client = new HttpClient(); + client.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); + return client; + } + + return Client; + } + + /// + /// blobStore detects the blob store for the given descriptor. + /// + /// + /// + private IBlobStore blobStore(Descriptor desc) + { + if (ManifestUtil.isManifest(ManifestMediaTypes, desc)) + { + return Manifests(); + } + + return Blobs(); + } + + + + /// + /// FetchAsync fetches the content identified by the descriptor. + /// + /// + /// + /// + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + return await blobStore(target).FetchAsync(target, cancellationToken); + } + + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + return await blobStore(target).ExistsAsync(target, cancellationToken); + } + + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + await blobStore(expected).PushAsync(expected, content, cancellationToken); + } + + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + /// + /// TagAsync tags a manifest descriptor with a reference string. + /// + /// + /// + /// + /// + public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) + { + await Manifests().TagAsync(descriptor, reference, cancellationToken); + } + + /// + /// FetchReference fetches the manifest identified by the reference. + /// The reference can be a tag or digest. + /// + /// + /// + /// + public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + { + return await Manifests().FetchReferenceAsync(reference, cancellationToken); + } + + /// + /// PushReference pushes the manifest with a reference tag. + /// + /// + /// + /// + /// + /// + public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, + CancellationToken cancellationToken = default) + { + await Manifests().FetchReferenceAsync(reference, cancellationToken); + } + + /// + /// DeleteAsync removes the content identified by the descriptor. + /// + /// + /// + /// + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + await blobStore(target).DeleteAsync(target, cancellationToken); + } + + public async Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + /// + /// Blobs provides access to the blob CAS only, which contains + /// layers, and other generic blobs. + /// + /// + public IBlobStore Blobs() + { + return new BlobStore(this); + } + + + /// + /// Manifests provides access to the + /// + /// + public IManifestStore Manifests() + { + return new ManifestStore(this); + } + + /// + /// ParseReference resolves a tag or a digest reference to a fully qualified + /// reference from a base reference Reference. + /// Tag, digest, or fully qualified references are accepted as input. + /// If reference is a fully qualified reference, then ParseReference parses it + /// and returns the parsed reference. If the parsed reference does not share + /// the same base reference with the Repository, ParseReference throws an + /// error, InvalidReferenceException. + /// + /// + /// + public ReferenceObj ParseReference(string reference) + { + try + { + var refObj = new ReferenceObj().ParseReference(reference); + if (refObj.Registry != Reference.Registry || refObj.Repository != Reference.Repository) + { + throw new InvalidReferenceException( + $"mismatch between received {JsonSerializer.Serialize(refObj)} and expected {JsonSerializer.Serialize(Reference)}"); + } + + if (refObj.Reference.Length == 0) + { + throw new InvalidReferenceException(); + } + return refObj; + } + catch (Exception) + { + var refObj = new ReferenceObj + { + Registry = Reference.Registry, + Repository = Reference.Repository, + Reference = Reference.Reference + }; + //reference is not a FQDN + if (reference.IndexOf("@") is var index && index != 1) + { + // `@` implies *digest*, so drop the *tag* (irrespective of what it is). + refObj.Reference = reference.Substring(index + 1); + refObj.ValidateReferenceAsDigest(); + } + else + { + refObj.ValidateReference(); + } + return refObj; + } + + } + } + + public class ManifestStore : IManifestStore + { + public Repository Repo { get; set; } + public ManifestStore(Repository repository) + { + Repo = repository; + } + + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + } + + public class BlobStore : IBlobStore + { + + public Repository Repo { get; set; } + + public BlobStore(Repository repository) + { + Repo = repository; + + } + + + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + } } From f1b96efbb15133c047d06536d04e34cb3159da77 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Tue, 9 May 2023 14:11:06 +0100 Subject: [PATCH 16/77] adding extra features Signed-off-by: Samson Amaugo --- Oras/Remote/Reference.cs | 13 +++++++++ Oras/Remote/RegistryUtil.cs | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 0498fd3..666a8f1 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -172,5 +172,18 @@ public void ValidateReference() ValidateReferenceAsTag(); } } + + /// + /// Host returns the host name of the registry + /// + /// + public string Host() + { + if (Registry == "docker.io") + { + return "registry-1.docker.io"; + } + return Registry; + } } } diff --git a/Oras/Remote/RegistryUtil.cs b/Oras/Remote/RegistryUtil.cs index 25ab301..346ae41 100644 --- a/Oras/Remote/RegistryUtil.cs +++ b/Oras/Remote/RegistryUtil.cs @@ -21,5 +21,59 @@ public static string BuildScheme(bool plainHTTP) return "https"; } + + /// + /// BuildRegistryBaseURL builds the URL for accessing the base API. + /// Format: :///v2/ + /// Reference: https://docs.docker.com/registry/spec/api/#base + /// + /// + /// + /// + public static string BuildRegistryBaseURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/"; + } + + /// + /// BuildManifestURL builds the URL for accessing the catalog API. + /// Format: :///v2/_catalog + /// Reference: https://docs.docker.com/registry/spec/api/#catalog + /// + /// + /// + /// + public static string BuildRegistryCatalogURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/_catalog"; + } + + /// + /// BuildRepositoryBaseURL builds the base endpoint of the remote repository. + /// Format: :///v2/ + /// + /// + /// + /// + public static string BuildRepositoryBaseURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}/"; + } + + /// + /// BuildRepositoryTagListURL builds the URL for accessing the tag list API. + /// Format: :///v2//tags/list + /// Reference: https://docs.docker.com/registry/spec/api/#tags + /// + /// + /// + /// + public static string BuildRepositoryTagListURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}/tags/list"; + } + + + } } From b17da8774102125f77ec1ffba3e5804cbf409800 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 11 May 2023 08:22:25 +0100 Subject: [PATCH 17/77] adding extra features Signed-off-by: Samson Amaugo --- Oras/Exceptions/NoLinkHeaderException.cs | 24 +++++ Oras/Remote/ErrorUtil.cs | 6 ++ Oras/Remote/Reference.cs | 4 +- Oras/Remote/RegistryUtil.cs | 50 +++++++-- Oras/Remote/Repository.cs | 124 ++++++++++++++++++++++- 5 files changed, 193 insertions(+), 15 deletions(-) create mode 100644 Oras/Exceptions/NoLinkHeaderException.cs create mode 100644 Oras/Remote/ErrorUtil.cs diff --git a/Oras/Exceptions/NoLinkHeaderException.cs b/Oras/Exceptions/NoLinkHeaderException.cs new file mode 100644 index 0000000..438d01c --- /dev/null +++ b/Oras/Exceptions/NoLinkHeaderException.cs @@ -0,0 +1,24 @@ +using System; + +namespace Oras.Exceptions +{ + /// + /// NoLinkHeaderException is thrown when a link header is missing. + /// + public class NoLinkHeaderException : Exception + { + public NoLinkHeaderException() + { + } + + public NoLinkHeaderException(string message) + : base(message) + { + } + + public NoLinkHeaderException(string message, Exception inner) + : base(message, inner) + { + } + } +} diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs new file mode 100644 index 0000000..e2b7a41 --- /dev/null +++ b/Oras/Remote/ErrorUtil.cs @@ -0,0 +1,6 @@ +namespace Oras.Remote +{ + internal class ErrorUtil + { + } +} diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 666a8f1..6d6721c 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -33,14 +33,14 @@ public class ReferenceObj /// Further checks are left to the server-side. /// References: /// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53 - /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#pulling-manifests + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// const string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; /// /// tagRegexp checks the tag name. /// The docker and OCI spec have the same regular expression. - /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#pulling-manifests + /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// const string tagRegexp = @"^[\w][\w.-]{0,127}$"; diff --git a/Oras/Remote/RegistryUtil.cs b/Oras/Remote/RegistryUtil.cs index 346ae41..79e1967 100644 --- a/Oras/Remote/RegistryUtil.cs +++ b/Oras/Remote/RegistryUtil.cs @@ -1,10 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Oras.Remote +namespace Oras.Remote { - + internal static class RegistryUtil { /// @@ -12,7 +8,7 @@ internal static class RegistryUtil /// /// /// - public static string BuildScheme(bool plainHTTP) + public static string BuildScheme(bool plainHTTP) { if (plainHTTP) { @@ -73,7 +69,45 @@ public static string BuildRepositoryTagListURL(bool plainHTTP, ReferenceObj refO return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}/tags/list"; } - + /// + /// BuildRepositoryManifestURL builds the URL for accessing the manifest API. + /// Format: :///v2//manifests/ + /// Reference: https://docs.docker.com/registry/spec/api/#manifest + /// + /// + /// + /// + public static string BuildRepositoryManifestURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/manifests/{refObj.Reference}"; + } + + /// + /// BuildRepositoryBlobURL builds the URL for accessing the blob API. + /// Format: :///v2//blobs/ + /// Reference: https://docs.docker.com/registry/spec/api/#blob + /// + /// + /// + /// + public static string BuildRepositoryBlobURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/blobs/{refObj.Reference}"; + } + + /// + /// BuildRepositoryBlobUploadURL builds the URL for accessing the blob upload API. + /// Format: :///v2//blobs/uploads/ + /// Reference: https://docs.docker.com/registry/spec/api/#initiate-blob-upload + + /// + /// + /// + /// + public static string BuildRepositoryBlobUploadURL(bool plainHTTP, ReferenceObj refObj) + { + return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/blobs/uploads/"; + } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index ea564d4..6ea2a01 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -1,12 +1,14 @@ -using Oras.Interfaces.Registry; +using Oras.Exceptions; +using Oras.Interfaces.Registry; using Oras.Models; using System; using System.IO; +using System.Linq; +using System.Net; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Oras.Exceptions; namespace Oras.Remote { @@ -64,6 +66,14 @@ public class Repository : IRepository, IRepositoryOption /// public long MaxMetadataBytes { get; set; } + /// + /// dockerContentDigestHeader - The Docker-Content-Digest header, if present + /// on the response, returns the canonical digest of the uploaded blob. + /// See https://docs.docker.com/registry/spec/api/#digest-header + /// See https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-digests + /// + public const string dockerContentDigestHeader = "Docker-Content-Digest" + /// /// Creates a client to the remote repository identified by a reference /// Example: localhost:5000/hello-world @@ -177,7 +187,7 @@ public async Task ResolveAsync(string reference, CancellationToken c /// /// public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) - { + { await Manifests().TagAsync(descriptor, reference, cancellationToken); } @@ -204,7 +214,7 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, CancellationToken cancellationToken = default) { - await Manifests().FetchReferenceAsync(reference, cancellationToken); + await Manifests().FetchReferenceAsync(reference, cancellationToken); } /// @@ -218,11 +228,112 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT await blobStore(target).DeleteAsync(target, cancellationToken); } + /// + /// TagsAsync lists the tags available in the repository. + /// See also `TagListPageSize`. + /// If `last` is NOT empty, the entries in the response start after the + /// tag specified by `last`. Otherwise, the response starts from the top + /// of the Tags list. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery + /// - https://docs.docker.com/registry/spec/api/#tags + /// + /// + /// + /// + /// public async Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default) + { + try + { + var url = RegistryUtil.BuildRepositoryTagListURL(PlainHTTP, Reference); + while (true) + { + await tagsAsync(last, fn, url, cancellationToken); + last = ""; + } + } + catch (Exception e) when (!(e is NoLinkHeaderException)) + { + + throw e; + + } + + } + + /// + /// tagsAsync returns a single page of tag list with the next link. + /// + /// + /// + /// + /// + /// + private async Task tagsAsync(string last, Action fn, string url, CancellationToken cancellationToken) { throw new NotImplementedException(); } + /// + /// deleteAsync removes the content identified by the descriptor in the + /// entity blobs or manifests. + /// + /// + /// + /// + /// + /// + /// + internal async Task deleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) + { + var refObj = Reference; + refObj.Reference = target.Digest; + Func buildURL = RegistryUtil.BuildRepositoryBlobURL; + if (isManifest) + { + buildURL = RegistryUtil.BuildRepositoryManifestURL; + } + + var url = buildURL(PlainHTTP, refObj); + var resp = await Client.DeleteAsync(url, cancellationToken); + + switch (resp.StatusCode) + { + case HttpStatusCode.Accepted: + return verifyContentDigest(resp, target.Digest); + case HttpStatusCode.NotFound: + throw new NotFoundException($"digest {target.Digest} not found"); + default: + throw new Exception(new + { + Method = resp.RequestMessage.Method, + URL = resp.RequestMessage.RequestUri, + StatusCode = resp.StatusCode + }.ToString()); + } + } + + + /// + /// verifyContentDigest verifies "Docker-Content-Digest" header if present. + /// OCI distribution-spec states the Docker-Content-Digest header is optional. + /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#legacy-docker-support-http-headers + /// + /// + /// + /// + private void verifyContentDigest(HttpResponseMessage resp, string targetDigest) + { + var digestStr = resp.Headers.GetValues(dockerContentDigestHeader); + if (!digestStr.Any()) + { + return; + } + var contentDigest = + } + + /// /// Blobs provides access to the blob CAS only, which contains /// layers, and other generic blobs. @@ -294,6 +405,8 @@ public ReferenceObj ParseReference(string reference) } } + + } public class ManifestStore : IManifestStore @@ -302,6 +415,7 @@ public class ManifestStore : IManifestStore public ManifestStore(Repository repository) { Repo = repository; + } public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) @@ -354,7 +468,7 @@ public class BlobStore : IBlobStore public BlobStore(Repository repository) { Repo = repository; - + } From 4f2279e32cfa3e938014959b577818abb5f257d1 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 11 May 2023 18:46:18 +0100 Subject: [PATCH 18/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/DigestUtil.cs | 23 ++++++++++ Oras/Remote/ErrorUtil.cs | 20 ++++++++- Oras/Remote/Reference.cs | 17 ++++---- Oras/Remote/Repository.cs | 83 +++++++++++++++++++++++++++++------- Oras/Remote/ResponseTypes.cs | 11 +++++ Oras/Remote/Utils.cs | 44 +++++++++++++++++++ 6 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 Oras/Remote/DigestUtil.cs create mode 100644 Oras/Remote/ResponseTypes.cs create mode 100644 Oras/Remote/Utils.cs diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Remote/DigestUtil.cs new file mode 100644 index 0000000..c1f4e71 --- /dev/null +++ b/Oras/Remote/DigestUtil.cs @@ -0,0 +1,23 @@ +using Oras.Exceptions; +using System.Text.RegularExpressions; + +namespace Oras.Remote +{ + internal class DigestUtil + { + /// + /// ParseDigest verifies the digest header and throws an exception if it is invalid. + /// + /// + /// + public static string Parse(string digest) + { + if (!Regex.IsMatch(digest, ReferenceObj.digestRegexp)) + { + throw new InvalidReferenceException($"invalid reference format: {digest}"); + } + + return digest; + } + } +} diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs index e2b7a41..ce2a7df 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtil.cs @@ -1,6 +1,24 @@ -namespace Oras.Remote +using System; +using System.Net.Http; + +namespace Oras.Remote { internal class ErrorUtil { + /// + /// ParseErrorResponse parses the error returned by the remote registry. + /// + /// + /// + public static void ParseErrorResponse(HttpResponseMessage resp) + { + throw new Exception(new + { + resp.RequestMessage.Method, + URL = resp.RequestMessage.RequestUri, + resp.StatusCode, + Errors = resp.Content.ReadAsStringAsync().Result + }.ToString()); + } } } diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 6d6721c..d0f5a72 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -35,19 +35,19 @@ public class ReferenceObj /// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53 /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - const string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; + public static string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; /// /// tagRegexp checks the tag name. /// The docker and OCI spec have the same regular expression. /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - const string tagRegexp = @"^[\w][\w.-]{0,127}$"; + public static string tagRegexp = @"^[\w][\w.-]{0,127}$"; /// /// digestRegexp checks the digest. /// - const string digestRegexp = @"^sha256:[0-9a-fA-F]{64}$"; + public static string digestRegexp = @"^sha256:[0-9a-fA-F]{64}$"; public ReferenceObj ParseReference(string artifact) { var parts = artifact.Split("/", 2); @@ -64,22 +64,22 @@ public ReferenceObj ParseReference(string artifact) { // digest found; Valid From A (if not B) isTag = false; - repository = path.Substring(0, index); - reference = path.Substring(index + 1); + repository = path[..index]; + reference = path[(index + 1)..]; if (repository.IndexOf(":") is var indexOfColon && indexOfColon != -1) { // tag found ( and now dropped without validation ) since the // digest already present; Valid Form B - repository = path.Substring(0, indexOfColon); + repository = repository[..index]; } } else if (path.IndexOf(":") is var indexOfColon && indexOfColon != -1) { // tag found; Valid Form C isTag = true; - repository = path.Substring(0, indexOfColon); - reference = path.Substring(indexOfColon + 1); + repository = path[..index]; + reference = path[(index + 1)..]; } else { @@ -121,6 +121,7 @@ public void ValidateReferenceAsDigest() } } + /// /// ValidateRepository checks if the repository name is valid. /// diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 6ea2a01..b193444 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -2,6 +2,7 @@ using Oras.Interfaces.Registry; using Oras.Models; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -72,8 +73,8 @@ public class Repository : IRepository, IRepositoryOption /// See https://docs.docker.com/registry/spec/api/#digest-header /// See https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-digests /// - public const string dockerContentDigestHeader = "Docker-Content-Digest" - + public const string dockerContentDigestHeader = "Docker-Content-Digest"; + /// /// Creates a client to the remote repository identified by a reference /// Example: localhost:5000/hello-world @@ -227,7 +228,18 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { await blobStore(target).DeleteAsync(target, cancellationToken); } + public async Task> TagsAsync(ITagLister repo, CancellationToken cancellationToken) + { + var res = new List(); + await repo.TagsAsync( + String.Empty, + async (tag) => + { + res.AddRange(tag); + }, cancellationToken); + return res; + } /// /// TagsAsync lists the tags available in the repository. /// See also `TagListPageSize`. @@ -272,7 +284,29 @@ public async Task TagsAsync(string last, Action fn, CancellationToken /// private async Task tagsAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - throw new NotImplementedException(); + if (TagListPageSize > 0 || last != "") + { + if (TagListPageSize > 0) + { + url = url + "?n=" + TagListPageSize; + } + if (last != "") + { + url = url + "&last=" + last; + } + } + var resp = await Client.GetAsync(url, cancellationToken); + if (resp.StatusCode != HttpStatusCode.OK) + { + ErrorUtil.ParseErrorResponse(resp); + + } + + var data = await resp.Content.ReadAsStringAsync(); + var page = JsonSerializer.Deserialize(data); + fn(page.tags); + return Utils.ParseLink(resp); + } /// @@ -301,16 +335,14 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation switch (resp.StatusCode) { case HttpStatusCode.Accepted: - return verifyContentDigest(resp, target.Digest); + verifyContentDigest(resp, target.Digest); + break; case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); + break; default: - throw new Exception(new - { - Method = resp.RequestMessage.Method, - URL = resp.RequestMessage.RequestUri, - StatusCode = resp.StatusCode - }.ToString()); + ErrorUtil.ParseErrorResponse(resp); + break; } } @@ -321,16 +353,37 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#legacy-docker-support-http-headers /// /// - /// + /// /// - private void verifyContentDigest(HttpResponseMessage resp, string targetDigest) + private void verifyContentDigest(HttpResponseMessage resp, string expected) { - var digestStr = resp.Headers.GetValues(dockerContentDigestHeader); - if (!digestStr.Any()) + var digestStr = resp.Headers.GetValues(dockerContentDigestHeader).FirstOrDefault(); + if (digestStr != null && !digestStr.Any()) { return; } - var contentDigest = + + string contentDigest; + try + { + contentDigest = DigestUtil.Parse(digestStr); + + } + catch (Exception e) + { + throw new Exception( + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: {dockerContentDigestHeader}: {digestStr}" + ); + } + + if (contentDigest != expected) + { + throw new Exception( +$"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in {dockerContentDigestHeader}: received {contentDigest}, while expecting {expected}" + ); + } + + return; } diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs new file mode 100644 index 0000000..07cb1d6 --- /dev/null +++ b/Oras/Remote/ResponseTypes.cs @@ -0,0 +1,11 @@ +namespace Oras.Remote +{ + internal class ResponseTypes + { + public class Tags + { + + public string[] tags { get; set; } + } + } +} diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs new file mode 100644 index 0000000..dd0ec44 --- /dev/null +++ b/Oras/Remote/Utils.cs @@ -0,0 +1,44 @@ +using Oras.Exceptions; +using System; +using System.Linq; +using System.Net.Http; + +namespace Oras.Remote +{ + internal class Utils + { + /// + /// ParseLink returns the URL of the response's "Link" header, if present. + /// + /// + /// + public static string ParseLink(HttpResponseMessage resp) + { + var link = resp.Headers.GetValues("Link").FirstOrDefault(); + if (String.IsNullOrEmpty(link)) + { + throw new NoLinkHeaderException(); + } + + if (link[0] != '<') + { + throw new Exception($"invalid next link {link}: missing '<"); + } + if (link.IndexOf('>') is var index && index < -1) + { + throw new Exception($"invalid next link {link}: missing '>'"); + } + else + { + link = link[1..index]; + } + + if (!Uri.IsWellFormedUriString(link, UriKind.Absolute)) + { + throw new Exception($"invalid next link {link}: not an absolute URL"); + } + + return link; + } + } +} From 0e97abd79bac13985cbfdfa224f81e68c956669e Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 11 May 2023 23:08:21 +0100 Subject: [PATCH 19/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index b193444..145d143 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -399,7 +399,7 @@ public IBlobStore Blobs() /// - /// Manifests provides access to the + /// Manifests provides access to the manifest CAS only. /// /// public IManifestStore Manifests() From 082d57dda547bb5033e8c705145e43561ba023b9 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 12 May 2023 06:09:11 +0100 Subject: [PATCH 20/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 3 +-- Oras/Remote/ResponseTypes.cs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 145d143..2c3f75d 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -339,7 +339,6 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation break; case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); - break; default: ErrorUtil.ParseErrorResponse(resp); break; @@ -369,7 +368,7 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) contentDigest = DigestUtil.Parse(digestStr); } - catch (Exception e) + catch (Exception) { throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: {dockerContentDigestHeader}: {digestStr}" diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs index 07cb1d6..1f943c9 100644 --- a/Oras/Remote/ResponseTypes.cs +++ b/Oras/Remote/ResponseTypes.cs @@ -4,7 +4,6 @@ internal class ResponseTypes { public class Tags { - public string[] tags { get; set; } } } From 5b3b1b4df5b43cd007ab57e90e5cebc0b10121a0 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 13 May 2023 13:19:51 +0100 Subject: [PATCH 21/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/ErrorUtil.cs | 6 +- Oras/Remote/Reference.cs | 44 ++++++++ Oras/Remote/Repository.cs | 224 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 262 insertions(+), 12 deletions(-) diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs index ce2a7df..1919d30 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtil.cs @@ -9,10 +9,10 @@ internal class ErrorUtil /// ParseErrorResponse parses the error returned by the remote registry. /// /// - /// - public static void ParseErrorResponse(HttpResponseMessage resp) + /// + public static Exception ParseErrorResponse(HttpResponseMessage resp) { - throw new Exception(new + return new Exception(new { resp.RequestMessage.Method, URL = resp.RequestMessage.RequestUri, diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index d0f5a72..fbc0369 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -1,5 +1,7 @@ using Oras.Exceptions; using System; +using System.Linq; +using System.Net.Http; using System.Text.RegularExpressions; namespace Oras.Remote @@ -186,5 +188,47 @@ public string Host() } return Registry; } + + /// + /// Digest returns the reference as a Digest + /// + /// + public string Digest() + { + ValidateReferenceAsDigest(); + return Reference; + } + + public static void VerifyContentDigest(HttpResponseMessage resp, string refDigest) + { + var digestStr = resp.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); + if (String.IsNullOrEmpty(digestStr)) + { + return; + } + + string contentDigest = String.Empty; + try + { + contentDigest = ParseDigest(digestStr); + } + catch (Exception e) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); + } + if (contentDigest != refDigest) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {refDigest}"); + } + } + + public static string ParseDigest(string digestStr) + { + if (!Regex.IsMatch(digestStr, digestRegexp)) + { + throw new InvalidReferenceException($"invalid reference format: {Reference}"); + } + return digestStr; + } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 2c3f75d..b1a867a 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Http.Headers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -175,9 +176,16 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok await blobStore(expected).PushAsync(expected, content, cancellationToken); } + /// + /// ResolveAsync resolves a reference to a manifest descriptor + /// See all ManifestMediaTypes + /// + /// + /// + /// public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + return await Manifests().ResolveAsync(reference, cancellationToken); } /// @@ -298,7 +306,7 @@ private async Task tagsAsync(string last, Action fn, string ur var resp = await Client.GetAsync(url, cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { - ErrorUtil.ParseErrorResponse(resp); + throw ErrorUtil.ParseErrorResponse(resp); } @@ -340,7 +348,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - ErrorUtil.ParseErrorResponse(resp); + throw ErrorUtil.ParseErrorResponse(resp); break; } } @@ -526,32 +534,230 @@ public BlobStore(Repository repository) public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.Reference; + refObj.Reference = target.Digest; + var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); + var resp = await Repo.Client.GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + // server does not support seek as `Range` was ignored. + if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); + } + + // check server range request capability. + // Docker spec allows range header form of "Range: bytes=-". + // However, the remote server may still not RFC 7233 compliant. + // Reference: https://docs.docker.com/registry/spec/api/#blob + if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") + { + var stream = new MemoryStream(); + long from = 0; + // make request using ranges until the whole data is read + while (from < target.Size) + { + + var to = from + 1024 * 1024 - 1; + if (to > target.Size) + { + to = target.Size; + } + Repo.Client.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); + resp = await Repo.Client.GetAsync(url, cancellationToken); + if (resp.StatusCode != HttpStatusCode.PartialContent) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); + } + await resp.Content.CopyToAsync(stream); + from = to + 1; + } + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + + return await resp.Content.ReadAsStreamAsync(); + case HttpStatusCode.NotFound: + throw new NotFoundException($"{target.Digest}: not found"); + default: + throw ErrorUtil.ParseErrorResponse(resp); + } } + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + try + { + await ResolveAsync(target.Digest, cancellationToken); + return true; + } + catch (Exception ex) when (ex is NotFoundException) + { + return false; + } + + catch (Exception ex) + { + throw ex; + } + } + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// Existing content is not checked by PushAsync() to minimize the number of out-going + /// requests. + /// Push is done by conventional 2-step monolithic upload instead of a single + /// `POST` request for better overall performance. It also allows early fail on + /// authentication errors. + /// References: + /// - https://docs.docker.com/registry/spec/api/#pushing-an-image + /// - https://docs.docker.com/registry/spec/api/#initiate-blob-upload + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pushing-a-blob-monolithically + /// + /// + /// + /// + /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + } + /// + /// ResolveAsync resolves a reference to a descriptor. + /// + /// + /// + /// public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + var refDigest = refObj.Digest(); + var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); + var resp = await Repo.Client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + return generateBlobDescriptor(resp, refDigest); + + case HttpStatusCode.NotFound: + throw new NotFoundException($"{refObj.Reference}: not found"); + default: + throw ErrorUtil.ParseErrorResponse(resp); + } } + /// + /// DeleteAsync deletes the content identified by the given descriptor. + /// + /// + /// + /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + await Repo.deleteAsync(target, false, cancellationToken); } + /// + /// FetchReferenceAsync fetches the blob identified by the reference. + /// The reference must be a digest. + /// + /// + /// + /// public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + var refDigest = refObj.Digest(); + var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); + var resp = await Repo.Client.GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.Accepted: + // server does not support seek as `Range` was ignored. + Descriptor desc = null; + if (resp.Content.Headers.ContentLength == -1) + { + desc = await ResolveAsync(refDigest, cancellationToken); + } + else + { + desc = generateBlobDescriptor(resp, refDigest); + } + // check server range request capability. + // Docker spec allows range header form of "Range: bytes=-". + // However, the remote server may still not RFC 7233 compliant. + // Reference: https://docs.docker.com/registry/spec/api/#blob + if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") + { + var stream = new MemoryStream(); + // make request using ranges until the whole data is read + long from = 0; + + while (from < desc.Size) + { + var to = from + 1024 * 1024 - 1; + if (to > desc.Size) + { + to = desc.Size; + } + Repo.Client.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); + resp = await Repo.Client.GetAsync(url, cancellationToken); + if (resp.StatusCode != HttpStatusCode.PartialContent) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); + } + await resp.Content.CopyToAsync(stream); + from = to + 1; + } + stream.Seek(0, SeekOrigin.Begin); + return (desc, stream); + } + return (desc, await resp.Content.ReadAsStreamAsync()); + case HttpStatusCode.NotFound: + throw new NotFoundException(); + default: + throw ErrorUtil.ParseErrorResponse(resp); + + } + } + + /// + /// generateBlobDescriptor returns a descriptor generated from the response. + /// + /// + /// + /// + /// + private Descriptor generateBlobDescriptor(HttpResponseMessage resp, string refDigest) + { + var mediaType = resp.Content.Headers.ContentType.MediaType; + if (String.IsNullOrEmpty(mediaType)) + { + mediaType = "application/octet-stream"; + } + var size = resp.Content.Headers.ContentLength!.Value; + if (size == -1) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); + } + + ReferenceObj.VerifyContentDigest(resp, refDigest); + + return new Descriptor + { + MediaType = mediaType, + Digest = refDigest, + Size = size + }; } } } From 46cc2421416d7511d299c80d657439e0e2976ec5 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 15 May 2023 06:02:27 +0100 Subject: [PATCH 22/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/DigestUtil.cs | 20 ++- Oras/Remote/ManifestUtil.cs | 12 +- Oras/Remote/Repository.cs | 335 +++++++++++++++++++++++++++++++++++- Oras/Remote/Utils.cs | 31 ++++ 4 files changed, 390 insertions(+), 8 deletions(-) diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Remote/DigestUtil.cs index c1f4e71..2f3362f 100644 --- a/Oras/Remote/DigestUtil.cs +++ b/Oras/Remote/DigestUtil.cs @@ -1,4 +1,7 @@ using Oras.Exceptions; +using System; +using System.Net.Http; +using System.Security.Cryptography; using System.Text.RegularExpressions; namespace Oras.Remote @@ -9,7 +12,6 @@ internal class DigestUtil /// ParseDigest verifies the digest header and throws an exception if it is invalid. /// /// - /// public static string Parse(string digest) { if (!Regex.IsMatch(digest, ReferenceObj.digestRegexp)) @@ -19,5 +21,21 @@ public static string Parse(string digest) return digest; } + + /// + /// Generates a digest from the content. + /// + /// + /// + public static string FromBytes(HttpContent content) + { + var digest = String.Empty; + using (var sha256 = SHA256.Create()) + { + var hash = sha256.ComputeHash(content.ReadAsByteArrayAsync().Result); + digest = $"sha256:{Convert.ToBase64String(hash)}"; + return digest; + } + } } } diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index aa4f69c..2c9cb23 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -20,7 +20,7 @@ static class ManifestUtil /// /// /// - public static bool isManifest(string[] manifestMediaTypes, Descriptor desc) + public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) { if (manifestMediaTypes.Length == 0) { @@ -34,5 +34,15 @@ public static bool isManifest(string[] manifestMediaTypes, Descriptor desc) return false; } + + public static string ManifestAcceptHeader(string[] manifestMediaTypes) + { + if (manifestMediaTypes.Length == 0) + { + manifestMediaTypes = DefaultManifestMediaTypes; + } + + return string.Join(",", manifestMediaTypes); + } } } \ No newline at end of file diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index b1a867a..152fbbe 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -11,6 +11,8 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using System.Web; +using Oras.Constants; namespace Oras.Remote { @@ -28,6 +30,13 @@ public class RepositoryOption : IRepositoryOption /// public class Repository : IRepository, IRepositoryOption { + /// + /// bytes are allowed in the server's response to the metadata APIs. + /// defaultMaxMetadataBytes specifies the default limit on how many response + /// See also: Repository.MaxMetadataBytes + /// + public long defaultMaxMetaBytes = 4 * 1024 * 1024; //4 Mib + /// /// Client is the underlying HTTP client used to access the remote registry. /// @@ -132,7 +141,7 @@ private HttpClient client() /// private IBlobStore blobStore(Descriptor desc) { - if (ManifestUtil.isManifest(ManifestMediaTypes, desc)) + if (ManifestUtil.IsManifest(ManifestMediaTypes, desc)) { return Manifests(); } @@ -478,24 +487,279 @@ public ManifestStore(Repository repository) } + /// + /// FetchASync fetches the content identified by the descriptor. + /// + /// + /// + /// + /// + /// public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.Reference; + refObj.Reference = target.Digest; + var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var req = new HttpRequestMessage(HttpMethod.Get, url); + req.Headers.Add("Accept", target.MediaType); + var resp = await Repo.Client.SendAsync(req, cancellationToken); + + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + break; + case HttpStatusCode.NotFound: + throw new NotFoundException($"digest {target.Digest} not found"); + default: + throw ErrorUtil.ParseErrorResponse(resp); + } + var mediaType = resp.Content.Headers.ContentType.MediaType; + if (mediaType != target.MediaType) + { + throw new Exception( + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch response Content-Type {mediaType}: expect {target.MediaType}"); + } + if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) + { + throw new Exception( + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); + } + ReferenceObj.VerifyContentDigest(resp, target.Digest); + return await resp.Content.ReadAsStreamAsync(); } + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + try + { + await ResolveAsync(target.Digest, cancellationToken); + return true; + } + catch (NotFoundException e) + { + return false; + } + + return false; + } + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// + /// + /// + /// + /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + pushWithIndexing(expected, content, expected.Digest, cancellationToken); + } + + /// + /// pushWithIndexing pushes the manifest content matching the expected descriptor. + /// + /// + /// + /// + /// + private void pushWithIndexing(Descriptor expected, Stream r, string reference, CancellationToken cancellationToken) + { + push(expected, r, reference, cancellationToken); + return; + } + + /// + /// push pushes the manifest content, matching the expected descriptor. + /// + /// + /// + /// + /// + private void push(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) + { + var refObj = Repo.Reference; + refObj.Reference = reference; + // pushing usually requires both pull and push actions. + // Reference: https://github.com/distribution/distribution/blob/v2.7.1/registry/handlers/app.go#L921-L930 + var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var req = new HttpRequestMessage(HttpMethod.Put, url); + req.Content = new StreamContent(stream); + if (req.Content != null && req.Content.Headers.ContentLength != expected.Size) + { + // short circuit a size mismatch for built-in types + throw new Exception( + $"{req.Method} {req.RequestUri}: mismatch Content-Length: expect {expected.Size}"); + } + req.Content.Headers.ContentLength = expected.Size; + req.Content.Headers.Add("Content-Type", expected.MediaType); + + // if the underlying client is an auth client, the content might be read + // more than once for obtaining the auth challenge and the actual request. + // To prevent double reading, the manifest is read and stored in the memory, + // and serve from the memory. + var client = Repo.Client; + var resp = client.SendAsync(req, cancellationToken).Result; + if (resp.StatusCode != HttpStatusCode.Created) + { + throw ErrorUtil.ParseErrorResponse(resp); + } + ReferenceObj.VerifyContentDigest(resp, expected.Digest); + } + + /// + /// limitSize returns ErrSizeExceedsLimit if the size of desc exceeds the limit n. + /// If n is less than or equal to zero, defaultMaxMetadataBytes is used. + /// + /// + /// + /// + private void limitSize(Descriptor desc, long n) + { + if (n <= 0) + { + n = Repo.defaultMaxMetaBytes; + } + + if (desc.Size > n) + { + throw new SizeExceedsLimitException($"content size {desc.Size} exceeds MaxMetadataBytes {n}"); + } + + return; } public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var req = new HttpRequestMessage(HttpMethod.Head, url); + req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); + var res = await Repo.Client.SendAsync(req, cancellationToken); + + switch (res.StatusCode) + { + case HttpStatusCode.OK: + return generateDescriptor(res, refObj, req.Method); + break; + case HttpStatusCode.NotFound: + throw new NotFoundException($"reference {reference} not found"); + default: + throw ErrorUtil.ParseErrorResponse(res); + } + } + + private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refObj, HttpMethod httpMethod) + { + try + { + // 1. Validate Content-Type + var mediaType = res.Content.Headers.ContentType.MediaType; + MediaTypeHeaderValue.TryParse(mediaType.ToString(), out var parsedMediaType); + + } + catch (Exception e) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response `Content-Type` header; {e.Message}"); + } + + // 2. Validate Size + if (!res.Content.Headers.ContentLength.HasValue) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: unknown response Content-Length"); + } + + // 3. Validate Client Reference + var refDigest = refObj.Digest(); + ReferenceObj.VerifyContentDigest(res, refObj.Digest()); + + // 4. Validate Server Digest (if present) + var serverHeaderDigest = res.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); + if (serverHeaderDigest != null) + { + try + { + ReferenceObj.VerifyContentDigest(res,serverHeaderDigest); + } + catch(Exception) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response header value: `Docker-Content-Digest: {serverHeaderDigest}`"); + } + } + + // 5. Now, look for specific error conditions; see truth table inmethod docstring + var contentDigest = String.Empty; + + if(serverHeaderDigest.Length == 0) + { + if (httpMethod == HttpMethod.Head) + { + if(refDigest.Length == 0) { + // HEAD without server `Docker-Content-Digest` + // immediate fail + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: HTTP {httpMethod} request missing required header {serverHeaderDigest}"); + } + // Otherwise, just trust the client-supplied digest + contentDigest = refDigest; + } + else + { + // GET without server `Docker-Content-Digest header forces the + // expensive calculation + var calculatedDigest = String.Empty; + try + { + calculatedDigest = calculateDigestFromResponse(res, Repo.MaxMetadataBytes); + } + catch (Exception e) + { + throw new Exception($"failed to calculate digest on response body; {e.Message}"); + } + contentDigest = calculatedDigest; + } + } + else + { + contentDigest = serverHeaderDigest; + } + if(refDigest.Length > 0 && refDigest != contentDigest) + { + /* + * + * return ocispec.Descriptor{}, fmt.Errorf( + "%s %q: invalid response; digest mismatch in %s: received %q when expecting %q", + resp.Request.Method, resp.Request.URL, + dockerContentDigestHeader, contentDigest, + refDigest, + ) + */ + + } + } + + /// + /// CalculateDigestFromResponse calculates the actual digest of the response body + /// taking care not to destroy it in the process + /// + /// + /// + private string calculateDigestFromResponse(HttpResponseMessage res, long maxMetadataBytes) + { + try + { + byte[] content = Utils.LimitReader(res.Content, maxMetadataBytes); + } + catch (Exception) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {e.Message}"); + } + return DigestUtil.FromBytes(res.Content); } public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) @@ -628,7 +892,66 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - + var url = RegistryUtil.BuildRepositoryBlobUploadURL(Repo.PlainHTTP, Repo.Reference); + var resp = await Repo.Client.PostAsync(url, null, cancellationToken); + var reqHostname = resp.RequestMessage.RequestUri.Host; + var reqPort = resp.RequestMessage.RequestUri.Port; + if (resp.StatusCode != HttpStatusCode.Accepted) + { + throw ErrorUtil.ParseErrorResponse(resp); + } + + // monolithic upload + var location = resp.Headers.Location; + // work-around solution for https://github.com/oras-project/oras-go/issues/177 + // For some registries, if the port 443 is explicitly set to the hostname + // like registry.wabbit-networks.io:443/myrepo, blob push will fail since + // the hostname of the Location header in the response is set to + // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. + + var locationHostname = location.Host; + var locationPort = location.Port; + // if location port 443 is missing, add it back + if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) + { + location = new Uri($"{locationHostname}:{reqPort}"); + } + url = location.ToString(); + + var req = new HttpRequestMessage(HttpMethod.Put, url); + req.Headers.Add("Content-Type", "application/octet-stream"); + req.Headers.Add("Content-Length", content.Length.ToString()); + req.Content = new StreamContent(content); + + if (req.Content != null && req.Content.Headers.ContentLength is var size && size != expected.Size) + { + throw new Exception($"mismatch content length {size}: expect {expected.Size}"); + } + req.Content.Headers.ContentLength = expected.Size; + + // the expected media type is ignored as in the API doc. + req.Headers.Add("Content-Type", "application/octet-stream"); + // add digest key to query string with expected digest value + var query = HttpUtility.ParseQueryString(location.Query); + query.Add("digest", expected.Digest); + req.RequestUri = new UriBuilder(location) + { + Query = query.ToString() + }.Uri; + + //reuse credential from previous POST request + var auth = resp.Headers.GetValues("Authorization").FirstOrDefault(); + if (auth != null) + { + req.Headers.Add("Authorization", auth); + } + resp = await Repo.Client.SendAsync(req, cancellationToken); + if (resp.StatusCode != HttpStatusCode.Created) + { + throw ErrorUtil.ParseErrorResponse(resp); + } + + return; } /// diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index dd0ec44..c4a73bf 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -7,6 +7,12 @@ namespace Oras.Remote { internal class Utils { + /// + /// defaultMaxMetadataBytes specifies the default limit on how many response + // bytes are allowed in the server's response to the metadata APIs. + // See also: Repository.MaxMetadataBytes + /// + const long defaultMaxMetadataBytes = 4 * 1024 * 1024; // 4 MiB /// /// ParseLink returns the URL of the response's "Link" header, if present. /// @@ -40,5 +46,30 @@ public static string ParseLink(HttpResponseMessage resp) return link; } + + /// + /// LimitReader returns a Reader that reads from content but stops after n + /// bytes. if n is less than or equal to zero, defaultMaxMetadataBytes is used. + /// + /// + /// + /// + /// + public static byte[] LimitReader(HttpContent content, long n) + { + if (n <= 0) + { + n = defaultMaxMetadataBytes; + } + + var bytes = content.ReadAsByteArrayAsync().Result; + + if (bytes.Length > n) + { + throw new Exception($"response body exceeds the limit of {n} bytes"); + } + + return bytes; + } } } From d8448c0e877cac77d6ed145555a006c79b724d86 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 15 May 2023 06:53:32 +0100 Subject: [PATCH 23/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/Reference.cs | 2 +- Oras/Remote/Repository.cs | 199 +++++++++++++++++++++++++------------- 2 files changed, 134 insertions(+), 67 deletions(-) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index fbc0369..9da838e 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -226,7 +226,7 @@ public static string ParseDigest(string digestStr) { if (!Regex.IsMatch(digestStr, digestRegexp)) { - throw new InvalidReferenceException($"invalid reference format: {Reference}"); + throw new InvalidReferenceException($"invalid reference format: {digestStr}"); } return digestStr; } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 152fbbe..8b86cbf 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -12,7 +12,6 @@ using System.Threading; using System.Threading.Tasks; using System.Web; -using Oras.Constants; namespace Oras.Remote { @@ -83,7 +82,7 @@ public class Repository : IRepository, IRepositoryOption /// See https://docs.docker.com/registry/spec/api/#digest-header /// See https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-digests /// - public const string dockerContentDigestHeader = "Docker-Content-Digest"; + public const string DockerContentDigestHeader = "Docker-Content-Digest"; /// /// Creates a client to the remote repository identified by a reference @@ -105,7 +104,7 @@ public Repository(string reference) /// /// /// - public Repository(ReferenceObj refObj, RepositoryOption option) + public Repository(ReferenceObj refObj, IRepositoryOption option) { refObj.ValidateRepository(); Client = option.Client; @@ -249,7 +248,7 @@ public async Task> TagsAsync(ITagLister repo, CancellationToken can { var res = new List(); await repo.TagsAsync( - String.Empty, + string.Empty, async (tag) => { res.AddRange(tag); @@ -315,7 +314,7 @@ private async Task tagsAsync(string last, Action fn, string ur var resp = await Client.GetAsync(url, cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { - throw ErrorUtil.ParseErrorResponse(resp); + throw ErrorUtil.ParseErrorResponse(resp); } @@ -357,7 +356,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw ErrorUtil.ParseErrorResponse(resp); break; } } @@ -373,7 +372,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// private void verifyContentDigest(HttpResponseMessage resp, string expected) { - var digestStr = resp.Headers.GetValues(dockerContentDigestHeader).FirstOrDefault(); + var digestStr = resp.Headers.GetValues(DockerContentDigestHeader).FirstOrDefault(); if (digestStr != null && !digestStr.Any()) { return; @@ -388,14 +387,14 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) catch (Exception) { throw new Exception( - $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: {dockerContentDigestHeader}: {digestStr}" + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: {DockerContentDigestHeader}: {digestStr}" ); } if (contentDigest != expected) { throw new Exception( -$"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in {dockerContentDigestHeader}: received {contentDigest}, while expecting {expected}" +$"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in {DockerContentDigestHeader}: received {contentDigest}, while expecting {expected}" ); } @@ -503,7 +502,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", target.MediaType); var resp = await Repo.Client.SendAsync(req, cancellationToken); - + switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -559,7 +558,7 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - pushWithIndexing(expected, content, expected.Digest, cancellationToken); + await pushWithIndexing(expected, content, expected.Digest, cancellationToken); } /// @@ -569,20 +568,20 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok /// /// /// - private void pushWithIndexing(Descriptor expected, Stream r, string reference, CancellationToken cancellationToken) + private async Task pushWithIndexing(Descriptor expected, Stream r, string reference, CancellationToken cancellationToken) { - push(expected, r, reference, cancellationToken); + await pushAsync(expected, r, reference, cancellationToken); return; } /// - /// push pushes the manifest content, matching the expected descriptor. + /// pushAsync pushes the manifest content, matching the expected descriptor. /// /// /// /// /// - private void push(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) + private async Task pushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) { var refObj = Repo.Reference; refObj.Reference = reference; @@ -605,7 +604,7 @@ private void push(Descriptor expected, Stream stream, string reference, Cancella // To prevent double reading, the manifest is read and stored in the memory, // and serve from the memory. var client = Repo.Client; - var resp = client.SendAsync(req, cancellationToken).Result; + var resp = await client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { throw ErrorUtil.ParseErrorResponse(resp); @@ -614,13 +613,13 @@ private void push(Descriptor expected, Stream stream, string reference, Cancella } /// - /// limitSize returns ErrSizeExceedsLimit if the size of desc exceeds the limit n. + /// LimitSize returns ErrSizeExceedsLimit if the size of desc exceeds the limit n. /// If n is less than or equal to zero, defaultMaxMetadataBytes is used. /// /// /// /// - private void limitSize(Descriptor desc, long n) + private void LimitSize(Descriptor desc, long n) { if (n <= 0) { @@ -655,13 +654,22 @@ public async Task ResolveAsync(string reference, CancellationToken c } } + /// + /// generateDescriptor returns a descriptor generated from the response. + /// + /// + /// + /// + /// + /// private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refObj, HttpMethod httpMethod) { + string mediaType; try { - // 1. Validate Content-Type - var mediaType = res.Content.Headers.ContentType.MediaType; - MediaTypeHeaderValue.TryParse(mediaType.ToString(), out var parsedMediaType); + // 1. Validate Content-Type + mediaType = res.Content.Headers.ContentType.MediaType; + MediaTypeHeaderValue.TryParse(mediaType.ToString(), out var parsedMediaType); } catch (Exception e) @@ -685,37 +693,38 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO { try { - ReferenceObj.VerifyContentDigest(res,serverHeaderDigest); + ReferenceObj.VerifyContentDigest(res, serverHeaderDigest); } - catch(Exception) + catch (Exception) { throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response header value: `Docker-Content-Digest: {serverHeaderDigest}`"); } } - // 5. Now, look for specific error conditions; see truth table inmethod docstring - var contentDigest = String.Empty; - - if(serverHeaderDigest.Length == 0) + // 5. Now, look for specific error conditions; + var contentDigest = string.Empty; + + if (serverHeaderDigest.Length == 0) { if (httpMethod == HttpMethod.Head) { - if(refDigest.Length == 0) { + if (refDigest.Length == 0) + { // HEAD without server `Docker-Content-Digest` // immediate fail throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: HTTP {httpMethod} request missing required header {serverHeaderDigest}"); } - // Otherwise, just trust the client-supplied digest - contentDigest = refDigest; + // Otherwise, just trust the client-supplied digest + contentDigest = refDigest; } else { // GET without server `Docker-Content-Digest header forces the // expensive calculation - var calculatedDigest = String.Empty; + string calculatedDigest; try { - calculatedDigest = calculateDigestFromResponse(res, Repo.MaxMetadataBytes); + calculatedDigest = calculateDigestFromResponse(res, Repo.MaxMetadataBytes); } catch (Exception e) { @@ -728,19 +737,18 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO { contentDigest = serverHeaderDigest; } - if(refDigest.Length > 0 && refDigest != contentDigest) + if (refDigest.Length > 0 && refDigest != contentDigest) { - /* - * - * return ocispec.Descriptor{}, fmt.Errorf( - "%s %q: invalid response; digest mismatch in %s: received %q when expecting %q", - resp.Request.Method, resp.Request.URL, - dockerContentDigestHeader, contentDigest, - refDigest, - ) - */ - + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response; digest mismatch in {serverHeaderDigest}: received {contentDigest} when expecting {refDigest}"); } + + // 6. Finally, if we made it this far, then all is good; return the descriptor + return new Descriptor + { + MediaType = mediaType, + Digest = contentDigest, + Size = res.Content.Headers.ContentLength.Value + }; } /// @@ -755,32 +763,96 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta { byte[] content = Utils.LimitReader(res.Content, maxMetadataBytes); } - catch (Exception) + catch (Exception ex) { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {e.Message}"); + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {ex.Message}"); } return DigestUtil.FromBytes(res.Content); } + /// + /// DeleteAsync removes the manifest content identified by the descriptor. + /// + /// + /// + /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + deleteWithIndexing(target, cancellationToken); + } + + /// + /// deleteWithIndexing removes the manifest content identified by the descriptor. + /// + /// + /// + /// + private async Task deleteWithIndexing(Descriptor target, CancellationToken cancellationToken) + { + await Repo.deleteAsync(target, true, cancellationToken); } + /// + /// FetchReferenceAsync fetches the manifest identified by the reference. + /// + /// + /// + /// public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var req = new HttpRequestMessage(HttpMethod.Get, url); + req.Content.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); + var resp = await Repo.Client.SendAsync(req, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + Descriptor desc; + if (resp.Content.Headers.ContentLength == -1) + { + desc = await ResolveAsync(reference, cancellationToken); + } + else + { + desc = generateDescriptor(resp, refObj, HttpMethod.Get); + } + return (desc, await resp.Content.ReadAsStreamAsync()); + case HttpStatusCode.NotFound: + throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); + default: + throw ErrorUtil.ParseErrorResponse(resp); + + } } - public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, + /// + /// PushReferenceASync pushes the manifest with a reference tag. + /// + /// + /// + /// + /// + /// + public async Task PushReferenceAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + await pushWithIndexing(expected, content, refObj.Reference, cancellationToken); } + /// + /// TagAsync tags a manifest descriptor with a reference string. + /// + /// + /// + /// + /// public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var refObj = Repo.ParseReference(reference); + var rc = await FetchAsync(descriptor, cancellationToken); + await pushAsync(descriptor, rc, refObj.Reference, cancellationToken); } } @@ -822,7 +894,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel // make request using ranges until the whole data is read while (from < target.Size) { - + var to = from + 1024 * 1024 - 1; if (to > target.Size) { @@ -845,7 +917,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw ErrorUtil.ParseErrorResponse(resp); } } @@ -867,11 +939,6 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell return false; } - catch (Exception ex) - { - throw ex; - } - } /// @@ -969,8 +1036,8 @@ public async Task ResolveAsync(string reference, CancellationToken c switch (resp.StatusCode) { case HttpStatusCode.OK: - return generateBlobDescriptor(resp, refDigest); - + return GenerateBlobDescriptor(resp, refDigest); + case HttpStatusCode.NotFound: throw new NotFoundException($"{refObj.Reference}: not found"); default: @@ -986,7 +1053,7 @@ public async Task ResolveAsync(string reference, CancellationToken c /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - await Repo.deleteAsync(target, false, cancellationToken); + await Repo.deleteAsync(target, false, cancellationToken); } /// @@ -1013,13 +1080,13 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT } else { - desc = generateBlobDescriptor(resp, refDigest); + desc = GenerateBlobDescriptor(resp, refDigest); } // check server range request capability. // Docker spec allows range header form of "Range: bytes=-". // However, the remote server may still not RFC 7233 compliant. // Reference: https://docs.docker.com/registry/spec/api/#blob - if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") + if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") { var stream = new MemoryStream(); // make request using ranges until the whole data is read @@ -1049,21 +1116,21 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT throw new NotFoundException(); default: throw ErrorUtil.ParseErrorResponse(resp); - + } } /// - /// generateBlobDescriptor returns a descriptor generated from the response. + /// GenerateBlobDescriptor returns a descriptor generated from the response. /// /// /// /// /// - private Descriptor generateBlobDescriptor(HttpResponseMessage resp, string refDigest) + private Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string refDigest) { var mediaType = resp.Content.Headers.ContentType.MediaType; - if (String.IsNullOrEmpty(mediaType)) + if (string.IsNullOrEmpty(mediaType)) { mediaType = "application/octet-stream"; } From 72f7c46ef6ba7cd6459ad5d8ecf7757675657401 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 15 May 2023 07:33:55 +0100 Subject: [PATCH 24/77] added extra features Signed-off-by: Samson Amaugo --- Oras/Remote/Registry.cs | 7 ------- Oras/Remote/Repository.cs | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 Oras/Remote/Registry.cs diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs deleted file mode 100644 index f9d7628..0000000 --- a/Oras/Remote/Registry.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Oras.Remote -{ - public class Registry - { - // public async Task Ping() - } -} diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 8b86cbf..5d23795 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -133,6 +133,30 @@ private HttpClient client() return Client; } + /// + /// PingAsync checks whether or not the registry implement Docker Registry API V2 or + /// OCI Distribution Specification. + /// Ping can be used to check authentication when an auth client is configured. + /// References: + /// - https://docs.docker.com/registry/spec/api/#base + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#api + /// + /// + /// + public async Task PingAsync(CancellationToken cancellationToken) + { + var url = RegistryUtil.BuildRegistryBaseURL(PlainHTTP, Reference); + var resp = await client().GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + return; + case HttpStatusCode.NotFound: + throw new NotFoundException($"Repository {Reference} not found"); + default: + throw ErrorUtil.ParseErrorResponse(resp); + } + } /// /// blobStore detects the blob store for the given descriptor. /// From ac4a1f7c919caeef7d1d0d4de43e9b4e51df524e Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 17 May 2023 20:04:03 +0100 Subject: [PATCH 25/77] testing the mic Signed-off-by: Samson Amaugo --- Oras/Oras.csproj | 3 ++- Oras/Program.cs | 28 ++++++++++++++++++++++++++++ Oras/Remote/Reference.cs | 24 ++++++++++++++---------- 3 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 Oras/Program.cs diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index 0876bf0..996650a 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -1,7 +1,8 @@  - netstandard2.1 + net7.0 + Exe diff --git a/Oras/Program.cs b/Oras/Program.cs new file mode 100644 index 0000000..a762742 --- /dev/null +++ b/Oras/Program.cs @@ -0,0 +1,28 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Oras.Remote; + +namespace Oras +{ + internal class Program + { + public static async Task Main() + { + //var repo = new Repository("localhost:5000/my-artifact:v1"); + var repo = new Repository(new ReferenceObj() + { + Reference = "v1", + Repository = "my-artifact", + Registry = "localhost:5000" + }, new RepositoryOption() + { + PlainHTTP = true + }); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + await repo.PingAsync(cancellationToken); + Console.WriteLine("Worked"); + } + } +} diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 9da838e..317e13f 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -73,15 +73,15 @@ public ReferenceObj ParseReference(string artifact) { // tag found ( and now dropped without validation ) since the // digest already present; Valid Form B - repository = repository[..index]; + repository = repository[..indexOfColon]; } } else if (path.IndexOf(":") is var indexOfColon && indexOfColon != -1) { // tag found; Valid Form C isTag = true; - repository = path[..index]; - reference = path[(index + 1)..]; + repository = path[..indexOfColon]; + reference = path[(indexOfColon + 1)..]; } else { @@ -94,18 +94,22 @@ public ReferenceObj ParseReference(string artifact) Repository = repository, Reference = reference }; - ValidateRegistry(); - ValidateRepository(); - if (Reference.Length == 0) + refObj.ValidateRegistry(); + refObj.ValidateRepository(); + + if (reference.Length== 0) { return refObj; } - ValidateReferenceAsDigest(); if (isTag) { - ValidateReferenceAsTag(); + refObj.ValidateReferenceAsTag(); + } + else + { + refObj.ValidateReferenceAsDigest(); } return refObj; } @@ -116,7 +120,7 @@ public ReferenceObj ParseReference(string artifact) public void ValidateReferenceAsDigest() { - + Console.WriteLine($"Reference is {Reference} and Digest is {digestRegexp}"); if (!Regex.IsMatch(Reference, digestRegexp)) { throw new InvalidReferenceException($"invalid reference format: {Reference}"); @@ -130,7 +134,7 @@ public void ValidateReferenceAsDigest() /// public void ValidateRepository() { - if (!Regex.IsMatch(Reference, repositoryRegexp)) + if (!Regex.IsMatch(Repository, repositoryRegexp)) { throw new InvalidReferenceException("Invalid Respository"); } From 740c59cb3c1257a592d5af5183ee335a5f10c721 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 17 May 2023 20:05:16 +0100 Subject: [PATCH 26/77] little fix Signed-off-by: Samson Amaugo --- Oras/Remote/Reference.cs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 9da838e..07de365 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -73,15 +73,15 @@ public ReferenceObj ParseReference(string artifact) { // tag found ( and now dropped without validation ) since the // digest already present; Valid Form B - repository = repository[..index]; + repository = repository[..indexOfColon]; } } else if (path.IndexOf(":") is var indexOfColon && indexOfColon != -1) { // tag found; Valid Form C isTag = true; - repository = path[..index]; - reference = path[(index + 1)..]; + repository = path[..indexOfColon]; + reference = path[(indexOfColon + 1)..]; } else { @@ -94,18 +94,22 @@ public ReferenceObj ParseReference(string artifact) Repository = repository, Reference = reference }; - ValidateRegistry(); - ValidateRepository(); - if (Reference.Length == 0) + refObj.ValidateRegistry(); + refObj.ValidateRepository(); + + if (reference.Length == 0) { return refObj; } - ValidateReferenceAsDigest(); if (isTag) { - ValidateReferenceAsTag(); + refObj.ValidateReferenceAsTag(); + } + else + { + refObj.ValidateReferenceAsDigest(); } return refObj; } @@ -116,7 +120,7 @@ public ReferenceObj ParseReference(string artifact) public void ValidateReferenceAsDigest() { - + Console.WriteLine($"Reference is {Reference} and Digest is {digestRegexp}"); if (!Regex.IsMatch(Reference, digestRegexp)) { throw new InvalidReferenceException($"invalid reference format: {Reference}"); @@ -130,7 +134,7 @@ public void ValidateReferenceAsDigest() /// public void ValidateRepository() { - if (!Regex.IsMatch(Reference, repositoryRegexp)) + if (!Regex.IsMatch(Repository, repositoryRegexp)) { throw new InvalidReferenceException("Invalid Respository"); } From 9de40d52323ab60029c3bee716dd199f191f7a7f Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 17 May 2023 20:22:01 +0100 Subject: [PATCH 27/77] little fix Signed-off-by: Samson Amaugo --- Oras/Remote/Reference.cs | 2 +- Oras/Remote/Repository.cs | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 07de365..8f2f9d7 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -216,7 +216,7 @@ public static void VerifyContentDigest(HttpResponseMessage resp, string refDiges { contentDigest = ParseDigest(digestStr); } - catch (Exception e) + catch (Exception) { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 5d23795..00d98c4 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -273,7 +273,7 @@ public async Task> TagsAsync(ITagLister repo, CancellationToken can var res = new List(); await repo.TagsAsync( string.Empty, - async (tag) => + (tag) => { res.AddRange(tag); @@ -381,7 +381,6 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation throw new NotFoundException($"digest {target.Digest} not found"); default: throw ErrorUtil.ParseErrorResponse(resp); - break; } } @@ -564,13 +563,11 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell await ResolveAsync(target.Digest, cancellationToken); return true; } - catch (NotFoundException e) + catch (NotFoundException) { return false; } - return false; - } /// @@ -670,7 +667,6 @@ public async Task ResolveAsync(string reference, CancellationToken c { case HttpStatusCode.OK: return generateDescriptor(res, refObj, req.Method); - break; case HttpStatusCode.NotFound: throw new NotFoundException($"reference {reference} not found"); default: @@ -802,16 +798,16 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - deleteWithIndexing(target, cancellationToken); + await deleteWithIndexingAsync(target, cancellationToken); } /// - /// deleteWithIndexing removes the manifest content identified by the descriptor. + /// deleteWithIndexingAsync removes the manifest content identified by the descriptor. /// /// /// /// - private async Task deleteWithIndexing(Descriptor target, CancellationToken cancellationToken) + private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken cancellationToken) { await Repo.deleteAsync(target, true, cancellationToken); } From fb5a37d51ca0a9b8683753bd095145e8067460e5 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 17 May 2023 20:26:09 +0100 Subject: [PATCH 28/77] little fix Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 00d98c4..6ac9f5f 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -305,11 +305,8 @@ public async Task TagsAsync(string last, Action fn, CancellationToken last = ""; } } - catch (Exception e) when (!(e is NoLinkHeaderException)) + catch (NoLinkHeaderException) { - - throw e; - } } From db94e93e7ad99380c2428eed0ed9f43aa65217ad Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 17 May 2023 20:28:29 +0100 Subject: [PATCH 29/77] little fix Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 00d98c4..da311b1 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -305,11 +305,9 @@ public async Task TagsAsync(string last, Action fn, CancellationToken last = ""; } } - catch (Exception e) when (!(e is NoLinkHeaderException)) + catch (NoLinkHeaderException) { - - throw e; - + return; } } From 136bf9bb724d56e55b6ddbe339e92374b52de458 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 18 May 2023 07:48:35 +0100 Subject: [PATCH 30/77] little fix Signed-off-by: Samson Amaugo --- Oras/Interfaces/Registry/IReferenceFetcher.cs | 2 +- Oras/Program.cs | 18 +++++++++++++++--- Oras/Remote/ManifestUtil.cs | 4 ++-- Oras/Remote/Repository.cs | 7 ++++--- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Oras/Interfaces/Registry/IReferenceFetcher.cs b/Oras/Interfaces/Registry/IReferenceFetcher.cs index b053cc1..d13a680 100644 --- a/Oras/Interfaces/Registry/IReferenceFetcher.cs +++ b/Oras/Interfaces/Registry/IReferenceFetcher.cs @@ -16,6 +16,6 @@ public interface IReferenceFetcher /// /// /// - Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); + Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); } } diff --git a/Oras/Program.cs b/Oras/Program.cs index a762742..4bc063f 100644 --- a/Oras/Program.cs +++ b/Oras/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Oras.Remote; @@ -14,15 +15,26 @@ public static async Task Main() { Reference = "v1", Repository = "my-artifact", - Registry = "localhost:5000" + Registry = "localhost:5000", + }, new RepositoryOption() { - PlainHTTP = true + PlainHTTP = true, + Client = new HttpClient() }); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); await repo.PingAsync(cancellationToken); - Console.WriteLine("Worked"); + var sha = "sha256:5461789af9cae6b426f19a4983cfd2cb8bbc4f0bd50fdaee9d3c8682c701787e"; + var sha2 = "sha256:91121276c3a3dbded7a94978dacaad6d992ba491921fa6f425d704c8ccc1a677"; + var resp = await repo.Blobs().FetchReferenceAsync(sha2, cancellationToken); + var bytes = new byte[resp.Descriptor.Size]; + await bytes.Stream.ReadAsync(bytes, 0, (int)resp.Descriptor.Size, cancellationToken); + // encode the byte to unicode + var str = System.Text.Encoding.UTF8.GetString(bytes); + Console.WriteLine(str); + + } } } diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index 2c9cb23..caf6997 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -22,7 +22,7 @@ static class ManifestUtil /// public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) { - if (manifestMediaTypes.Length == 0) + if (manifestMediaTypes == null || manifestMediaTypes.Length == 0) { manifestMediaTypes = DefaultManifestMediaTypes; } @@ -37,7 +37,7 @@ public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) public static string ManifestAcceptHeader(string[] manifestMediaTypes) { - if (manifestMediaTypes.Length == 0) + if (manifestMediaTypes == null || manifestMediaTypes.Length == 0 ) { manifestMediaTypes = DefaultManifestMediaTypes; } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 6ac9f5f..c17bea4 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -307,6 +307,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken } catch (NoLinkHeaderException) { + return; } } @@ -820,7 +821,7 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken var refObj = Repo.ParseReference(reference); var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Content.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); + req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); var resp = await Repo.Client.SendAsync(req, cancellationToken); switch (resp.StatusCode) { @@ -1080,7 +1081,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT /// /// /// - public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { var refObj = Repo.ParseReference(reference); var refDigest = refObj.Digest(); @@ -1088,7 +1089,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var resp = await Repo.Client.GetAsync(url, cancellationToken); switch (resp.StatusCode) { - case HttpStatusCode.Accepted: + case HttpStatusCode.OK: // server does not support seek as `Range` was ignored. Descriptor desc = null; if (resp.Content.Headers.ContentLength == -1) From ea516035d4a2aebd8d1f5a5408b0a777b8f2dd13 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 18 May 2023 07:54:55 +0100 Subject: [PATCH 31/77] created client code Signed-off-by: Samson Amaugo --- Oras/Oras.csproj | 5 ++--- Oras/Program.cs | 40 ---------------------------------------- Oras/Remote/Reference.cs | 2 -- 3 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 Oras/Program.cs diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index 996650a..015f973 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -1,8 +1,7 @@  - net7.0 - Exe + netstandard2.1 @@ -12,4 +11,4 @@ - + \ No newline at end of file diff --git a/Oras/Program.cs b/Oras/Program.cs deleted file mode 100644 index 4bc063f..0000000 --- a/Oras/Program.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Oras.Remote; - -namespace Oras -{ - internal class Program - { - public static async Task Main() - { - //var repo = new Repository("localhost:5000/my-artifact:v1"); - var repo = new Repository(new ReferenceObj() - { - Reference = "v1", - Repository = "my-artifact", - Registry = "localhost:5000", - - }, new RepositoryOption() - { - PlainHTTP = true, - Client = new HttpClient() - }); - repo.PlainHTTP = true; - var cancellationToken = new CancellationToken(); - await repo.PingAsync(cancellationToken); - var sha = "sha256:5461789af9cae6b426f19a4983cfd2cb8bbc4f0bd50fdaee9d3c8682c701787e"; - var sha2 = "sha256:91121276c3a3dbded7a94978dacaad6d992ba491921fa6f425d704c8ccc1a677"; - var resp = await repo.Blobs().FetchReferenceAsync(sha2, cancellationToken); - var bytes = new byte[resp.Descriptor.Size]; - await bytes.Stream.ReadAsync(bytes, 0, (int)resp.Descriptor.Size, cancellationToken); - // encode the byte to unicode - var str = System.Text.Encoding.UTF8.GetString(bytes); - Console.WriteLine(str); - - - } - } -} diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 8f2f9d7..28f10c0 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -119,8 +119,6 @@ public ReferenceObj ParseReference(string artifact) /// public void ValidateReferenceAsDigest() { - - Console.WriteLine($"Reference is {Reference} and Digest is {digestRegexp}"); if (!Regex.IsMatch(Reference, digestRegexp)) { throw new InvalidReferenceException($"invalid reference format: {Reference}"); From acf852b3fc82505054f4212bdbcf3d9b3a8f4112 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 18 May 2023 07:55:42 +0100 Subject: [PATCH 32/77] created client code Signed-off-by: Samson Amaugo --- Oras/Oras.csproj | 1 - Oras/Remote/ManifestUtil.cs | 2 +- Oras/Remote/Repository.cs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index 015f973..d588c19 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -1,5 +1,4 @@  - netstandard2.1 diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index caf6997..b216d99 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -37,7 +37,7 @@ public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) public static string ManifestAcceptHeader(string[] manifestMediaTypes) { - if (manifestMediaTypes == null || manifestMediaTypes.Length == 0 ) + if (manifestMediaTypes == null || manifestMediaTypes.Length == 0) { manifestMediaTypes = DefaultManifestMediaTypes; } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index c17bea4..4003b1a 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -796,7 +796,7 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - await deleteWithIndexingAsync(target, cancellationToken); + await deleteWithIndexingAsync(target, cancellationToken); } /// From 727022f807116d6ea65061578da881dc3d640a1d Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 18 May 2023 08:01:25 +0100 Subject: [PATCH 33/77] made some fixes Signed-off-by: Samson Amaugo --- Oras/Remote/DigestUtil.cs | 4 ++-- Oras/Remote/ManifestUtil.cs | 5 +++++ Oras/Remote/Reference.cs | 2 +- Oras/Remote/Utils.cs | 5 +++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Remote/DigestUtil.cs index 2f3362f..ecf4c6c 100644 --- a/Oras/Remote/DigestUtil.cs +++ b/Oras/Remote/DigestUtil.cs @@ -9,7 +9,7 @@ namespace Oras.Remote internal class DigestUtil { /// - /// ParseDigest verifies the digest header and throws an exception if it is invalid. + /// Parse verifies the digest header and throws an exception if it is invalid. /// /// public static string Parse(string digest) @@ -23,7 +23,7 @@ public static string Parse(string digest) } /// - /// Generates a digest from the content. + /// FromBytes generates a digest from the content. /// /// /// diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index b216d99..09ec95a 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -35,6 +35,11 @@ public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) return false; } + /// + /// ManifestAcceptHeader returns the accept header for the given manifest media types. + /// + /// + /// public static string ManifestAcceptHeader(string[] manifestMediaTypes) { if (manifestMediaTypes == null || manifestMediaTypes.Length == 0) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 28f10c0..7f25f9f 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -19,7 +19,7 @@ public class ReferenceObj public string Repository { get; set; } /// - /// Ref is the reference of the object in the repository. This field + /// Reference is the reference of the object in the repository. This field /// can take any one of the four valid forms (see ParseReference). In the /// case where it's the empty string, it necessarily implies valid form D, /// and where it is non-empty, then it is either a tag, or a digest diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index c4a73bf..adf8fb2 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -9,10 +9,11 @@ internal class Utils { /// /// defaultMaxMetadataBytes specifies the default limit on how many response - // bytes are allowed in the server's response to the metadata APIs. - // See also: Repository.MaxMetadataBytes + /// bytes are allowed in the server's response to the metadata APIs. + /// See also: Repository.MaxMetadataBytes /// const long defaultMaxMetadataBytes = 4 * 1024 * 1024; // 4 MiB + /// /// ParseLink returns the URL of the response's "Link" header, if present. /// From 5953bb88e47da745b6cf1d8db2f0307640b786c7 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 04:36:57 +0100 Subject: [PATCH 34/77] added test Signed-off-by: Samson Amaugo --- Oras.Tests/Oras.Tests.csproj | 1 + Oras.Tests/RemoteTest/RemoteTest.cs | 102 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 Oras.Tests/RemoteTest/RemoteTest.cs diff --git a/Oras.Tests/Oras.Tests.csproj b/Oras.Tests/Oras.Tests.csproj index 7ec6cb3..f15a874 100644 --- a/Oras.Tests/Oras.Tests.csproj +++ b/Oras.Tests/Oras.Tests.csproj @@ -10,6 +10,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs new file mode 100644 index 0000000..f30787e --- /dev/null +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using static Oras.Content.Content; +using System.Threading.Tasks; +using Moq; +using Moq.Protected; +using Oras.Constants; +using Oras.Models; +using Oras.Remote; +using Xunit; + +namespace Oras.Tests.RemoteTest +{ + public class RemoteTest + { + public HttpClient CustomClient(Func func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).ReturnsAsync(func); + return new HttpClient(moqHandler.Object); + } + + [Fact] + public async Task Repository_FetchAsync() + { + var blob = Encoding.UTF8.GetBytes("hello world"); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint) blob.Length + }; + var index = Encoding.UTF8.GetBytes("""{"manifests":[]}"""); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + if (req.Method != HttpMethod.Get) + { + Debug.WriteLine("Expected GET request"); + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var path = req.RequestUri!.AbsolutePath; + if (path == "v2/test/blobs/" + blobDesc.Digest) + { + resp.Headers.Add("Content-Type", "application/octet-stream"); + resp.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + resp.Content = new ByteArrayContent(blob); + return resp; + } + else if (path == "v2/test/manifests/" + indexDesc.Digest) + { + if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(OCIMediaTypes.ImageIndex))) + { + resp.StatusCode = HttpStatusCode.BadRequest; + Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); + return resp; + } + + resp.Headers.Add("Content-Type", indexDesc.MediaType); + resp.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + resp.Content = new ByteArrayContent(index); + return resp; + + } + else + { + Debug.WriteLine("Unexpected path: " + path); + resp.StatusCode = HttpStatusCode.NotFound; + return resp; + } + + }; + var repo = new Repository("http://localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var stream = await repo.FetchAsync(blobDesc,cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, 0, (int)stream.Length, cancellationToken); + Assert.Equal(blob, buf); + + + } + } +} From 5535e1d71df87fa35ea583550a0bad26d18a4072 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 05:25:59 +0100 Subject: [PATCH 35/77] added test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 14 ++++---- Oras/Remote/DigestUtil.cs | 11 +++--- Oras/Remote/Reference.cs | 2 +- Oras/Remote/RegistryUtil.cs | 2 +- Oras/Remote/Repository.cs | 53 +++++++++++++---------------- 5 files changed, 36 insertions(+), 46 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index f30787e..1a6e3d3 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -18,7 +18,7 @@ namespace Oras.Tests.RemoteTest { public class RemoteTest { - public HttpClient CustomClient(Func func) + public static HttpClient CustomClient(Func func) { var moqHandler = new Mock(); moqHandler.Protected().Setup>( @@ -57,14 +57,14 @@ public async Task Repository_FetchAsync() } var path = req.RequestUri!.AbsolutePath; - if (path == "v2/test/blobs/" + blobDesc.Digest) + if (path == "/v2/test/blobs/" + blobDesc.Digest) { - resp.Headers.Add("Content-Type", "application/octet-stream"); - resp.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + resp.Content.Headers.Add("Content-Type", "application/octet-stream"); + resp.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); resp.Content = new ByteArrayContent(blob); return resp; } - else if (path == "v2/test/manifests/" + indexDesc.Digest) + else if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(OCIMediaTypes.ImageIndex))) { @@ -87,13 +87,13 @@ public async Task Repository_FetchAsync() } }; - var repo = new Repository("http://localhost:5000/test"); + var repo = new Repository("localhost:5000/test"); repo.Client = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc,cancellationToken); var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, 0, (int)stream.Length, cancellationToken); + await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Remote/DigestUtil.cs index ecf4c6c..7f7dc6a 100644 --- a/Oras/Remote/DigestUtil.cs +++ b/Oras/Remote/DigestUtil.cs @@ -29,13 +29,10 @@ public static string Parse(string digest) /// public static string FromBytes(HttpContent content) { - var digest = String.Empty; - using (var sha256 = SHA256.Create()) - { - var hash = sha256.ComputeHash(content.ReadAsByteArrayAsync().Result); - digest = $"sha256:{Convert.ToBase64String(hash)}"; - return digest; - } + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(content.ReadAsByteArrayAsync().Result); + var digest = $"sha256:{Convert.ToBase64String(hash)}"; + return digest; } } } diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 7f25f9f..96d5182 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -209,7 +209,7 @@ public static void VerifyContentDigest(HttpResponseMessage resp, string refDiges return; } - string contentDigest = String.Empty; + string contentDigest; try { contentDigest = ParseDigest(digestStr); diff --git a/Oras/Remote/RegistryUtil.cs b/Oras/Remote/RegistryUtil.cs index 79e1967..fbe222e 100644 --- a/Oras/Remote/RegistryUtil.cs +++ b/Oras/Remote/RegistryUtil.cs @@ -53,7 +53,7 @@ public static string BuildRegistryCatalogURL(bool plainHTTP, ReferenceObj refObj /// public static string BuildRepositoryBaseURL(bool plainHTTP, ReferenceObj refObj) { - return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}/"; + return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}"; } /// diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 4003b1a..14146da 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -393,7 +393,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// private void verifyContentDigest(HttpResponseMessage resp, string expected) { - var digestStr = resp.Headers.GetValues(DockerContentDigestHeader).FirstOrDefault(); + resp.Headers.TryGetValues(DockerContentDigestHeader,out var digestStr); if (digestStr != null && !digestStr.Any()) { return; @@ -402,7 +402,7 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) string contentDigest; try { - contentDigest = DigestUtil.Parse(digestStr); + contentDigest = DigestUtil.Parse(digestStr.FirstOrDefault()); } catch (Exception) @@ -483,7 +483,7 @@ public ReferenceObj ParseReference(string reference) if (reference.IndexOf("@") is var index && index != 1) { // `@` implies *digest*, so drop the *tag* (irrespective of what it is). - refObj.Reference = reference.Substring(index + 1); + refObj.Reference = reference[(index + 1)..]; refObj.ValidateReferenceAsDigest(); } else @@ -661,15 +661,12 @@ public async Task ResolveAsync(string reference, CancellationToken c req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); var res = await Repo.Client.SendAsync(req, cancellationToken); - switch (res.StatusCode) + return res.StatusCode switch { - case HttpStatusCode.OK: - return generateDescriptor(res, refObj, req.Method); - case HttpStatusCode.NotFound: - throw new NotFoundException($"reference {reference} not found"); - default: - throw ErrorUtil.ParseErrorResponse(res); - } + HttpStatusCode.OK => generateDescriptor(res, refObj, req.Method), + HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), + _ => throw ErrorUtil.ParseErrorResponse(res) + }; } /// @@ -706,12 +703,12 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO ReferenceObj.VerifyContentDigest(res, refObj.Digest()); // 4. Validate Server Digest (if present) - var serverHeaderDigest = res.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); + res.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); if (serverHeaderDigest != null) { try { - ReferenceObj.VerifyContentDigest(res, serverHeaderDigest); + ReferenceObj.VerifyContentDigest(res, serverHeaderDigest.FirstOrDefault()); } catch (Exception) { @@ -720,9 +717,9 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO } // 5. Now, look for specific error conditions; - var contentDigest = string.Empty; + string contentDigest; - if (serverHeaderDigest.Length == 0) + if (serverHeaderDigest.FirstOrDefault().Length == 0) { if (httpMethod == HttpMethod.Head) { @@ -753,7 +750,7 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO } else { - contentDigest = serverHeaderDigest; + contentDigest = serverHeaderDigest.FirstOrDefault(); } if (refDigest.Length > 0 && refDigest != contentDigest) { @@ -905,7 +902,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel // Docker spec allows range header form of "Range: bytes=-". // However, the remote server may still not RFC 7233 compliant. // Reference: https://docs.docker.com/registry/spec/api/#blob - if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") + if (resp.Headers.TryGetValues("Accept-Ranges", out var acceptRanges) && acceptRanges.FirstOrDefault() == "bytes") { var stream = new MemoryStream(); long from = 0; @@ -1025,10 +1022,10 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok }.Uri; //reuse credential from previous POST request - var auth = resp.Headers.GetValues("Authorization").FirstOrDefault(); + resp.Headers.TryGetValues("Authorization", out var auth); if (auth != null) { - req.Headers.Add("Authorization", auth); + req.Headers.Add("Authorization", auth.FirstOrDefault()); } resp = await Repo.Client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) @@ -1051,16 +1048,12 @@ public async Task ResolveAsync(string reference, CancellationToken c var refDigest = refObj.Digest(); var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); var resp = await Repo.Client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - switch (resp.StatusCode) + return resp.StatusCode switch { - case HttpStatusCode.OK: - return GenerateBlobDescriptor(resp, refDigest); - - case HttpStatusCode.NotFound: - throw new NotFoundException($"{refObj.Reference}: not found"); - default: - throw ErrorUtil.ParseErrorResponse(resp); - } + HttpStatusCode.OK => GenerateBlobDescriptor(resp, refDigest), + HttpStatusCode.NotFound => throw new NotFoundException($"{refObj.Reference}: not found"), + _ => throw ErrorUtil.ParseErrorResponse(resp) + }; } /// @@ -1091,7 +1084,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { case HttpStatusCode.OK: // server does not support seek as `Range` was ignored. - Descriptor desc = null; + Descriptor desc; if (resp.Content.Headers.ContentLength == -1) { desc = await ResolveAsync(refDigest, cancellationToken); @@ -1104,7 +1097,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT // Docker spec allows range header form of "Range: bytes=-". // However, the remote server may still not RFC 7233 compliant. // Reference: https://docs.docker.com/registry/spec/api/#blob - if (resp.Headers.GetValues("Accept-Ranges").FirstOrDefault() == "bytes") + if (resp.Headers.TryGetValues("Accept-Ranges", out var acceptRanges) && acceptRanges.FirstOrDefault() == "bytes") { var stream = new MemoryStream(); // make request using ranges until the whole data is read From 500b78c755cbb23622535ae2f614a561bb90dd41 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 06:06:24 +0100 Subject: [PATCH 36/77] made some fixes Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 11 +++++++---- Oras/Remote/Reference.cs | 2 +- Oras/Remote/Repository.cs | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 1a6e3d3..2d55f23 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -59,9 +59,9 @@ public async Task Repository_FetchAsync() var path = req.RequestUri!.AbsolutePath; if (path == "/v2/test/blobs/" + blobDesc.Digest) { + resp.Content = new ByteArrayContent(blob); resp.Content.Headers.Add("Content-Type", "application/octet-stream"); resp.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); - resp.Content = new ByteArrayContent(blob); return resp; } else if (path == "/v2/test/manifests/" + indexDesc.Digest) @@ -73,9 +73,9 @@ public async Task Repository_FetchAsync() return resp; } - resp.Headers.Add("Content-Type", indexDesc.MediaType); - resp.Headers.Add("Docker-Content-Digest", indexDesc.Digest); resp.Content = new ByteArrayContent(index); + resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); + resp.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); return resp; } @@ -95,7 +95,10 @@ public async Task Repository_FetchAsync() var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); - + stream = await repo.FetchAsync(indexDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); } } diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 96d5182..84cbce3 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -203,7 +203,7 @@ public string Digest() public static void VerifyContentDigest(HttpResponseMessage resp, string refDigest) { - var digestStr = resp.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); + var digestStr = resp.Content.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); if (String.IsNullOrEmpty(digestStr)) { return; diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 14146da..2aa6b21 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -533,13 +533,13 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel default: throw ErrorUtil.ParseErrorResponse(resp); } - var mediaType = resp.Content.Headers.ContentType.MediaType; + var mediaType = resp.Content.Headers?.ContentType.MediaType; if (mediaType != target.MediaType) { throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch response Content-Type {mediaType}: expect {target.MediaType}"); } - if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) + if (resp.Content.Headers!.ContentLength is var size && size != -1 && size != target.Size) { throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); From 287141172a8123751cf7cb1affbd3750e2b655a0 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 09:06:25 +0100 Subject: [PATCH 37/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 128 +++++++++++++++++++++++----- Oras/Remote/Repository.cs | 13 +-- 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 2d55f23..f6c850d 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1,18 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using static Oras.Content.Content; -using System.Threading.Tasks; -using Moq; +using Moq; using Moq.Protected; using Oras.Constants; using Oras.Models; using Oras.Remote; +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using System.Text; using Xunit; +using static Oras.Content.Content; namespace Oras.Tests.RemoteTest { @@ -29,6 +25,10 @@ public static HttpClient CustomClient(Func + /// Repository_FetchAsync tests the FetchAsync method of the Repository. + /// + /// [Fact] public async Task Repository_FetchAsync() { @@ -37,9 +37,9 @@ public async Task Repository_FetchAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; - var index = Encoding.UTF8.GetBytes("""{"manifests":[]}"""); + var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -64,7 +64,7 @@ public async Task Repository_FetchAsync() resp.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return resp; } - else if (path == "/v2/test/manifests/" + indexDesc.Digest) + if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(OCIMediaTypes.ImageIndex))) { @@ -72,26 +72,20 @@ public async Task Repository_FetchAsync() Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } - resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); return resp; - - } - else - { - Debug.WriteLine("Unexpected path: " + path); - resp.StatusCode = HttpStatusCode.NotFound; - return resp; - } + } + resp.StatusCode = HttpStatusCode.NotFound; + return resp; }; var repo = new Repository("localhost:5000/test"); repo.Client = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); - var stream = await repo.FetchAsync(blobDesc,cancellationToken); + var stream = await repo.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); @@ -101,5 +95,93 @@ public async Task Repository_FetchAsync() Assert.Equal(index, buf); } + + /// + /// Repository_PushAsync tests the PushAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushAsync() + { + var blob = Encoding.UTF8.GetBytes("hello world"); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var uuid = Guid.NewGuid().ToString(); + var gotBlob = new byte[blobDesc.Size]; + var gotIndex = new byte[indexDesc.Size]; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") + { + resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); + resp.StatusCode = HttpStatusCode.Accepted; + return resp; + } + + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + + } + if (!req.RequestUri.Query.Contains("digest=" + blobDesc.Digest)) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + resp.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + + } + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotIndex); + resp.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + } + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); + Assert.Equal(index, gotIndex); + } } } + diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 2aa6b21..b24cc3b 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -393,7 +393,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// private void verifyContentDigest(HttpResponseMessage resp, string expected) { - resp.Headers.TryGetValues(DockerContentDigestHeader,out var digestStr); + resp.Headers.TryGetValues(DockerContentDigestHeader, out var digestStr); if (digestStr != null && !digestStr.Any()) { return; @@ -1001,10 +1001,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok url = location.ToString(); var req = new HttpRequestMessage(HttpMethod.Put, url); - req.Headers.Add("Content-Type", "application/octet-stream"); - req.Headers.Add("Content-Length", content.Length.ToString()); req.Content = new StreamContent(content); - if (req.Content != null && req.Content.Headers.ContentLength is var size && size != expected.Size) { throw new Exception($"mismatch content length {size}: expect {expected.Size}"); @@ -1012,14 +1009,12 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok req.Content.Headers.ContentLength = expected.Size; // the expected media type is ignored as in the API doc. - req.Headers.Add("Content-Type", "application/octet-stream"); + req.Content.Headers.Add("Content-Type", "application/octet-stream"); + // add digest key to query string with expected digest value var query = HttpUtility.ParseQueryString(location.Query); query.Add("digest", expected.Digest); - req.RequestUri = new UriBuilder(location) - { - Query = query.ToString() - }.Uri; + req.RequestUri = new Uri($"{req.RequestUri}?digest={expected.Digest}"); //reuse credential from previous POST request resp.Headers.TryGetValues("Authorization", out var auth); From 5a9060b0ebb341b5b834bb99dcbcd0163124f708 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 16:55:10 +0100 Subject: [PATCH 38/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index f6c850d..bdf0b83 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -103,7 +103,7 @@ public async Task Repository_FetchAsync() [Fact] public async Task Repository_PushAsync() { - var blob = Encoding.UTF8.GetBytes("hello world"); + var blob = @"""hello world"""u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), @@ -182,6 +182,25 @@ public async Task Repository_PushAsync() await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } + + public async Task Repository_ExistsAsync() + { + var blob = @"""hello world"""u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + + } } } From fdca67961eb1d6c56679a41862f0d82bc5dac8fa Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 19 May 2023 18:44:45 +0100 Subject: [PATCH 39/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 92 +++++++++++++++++++++++++++++ Oras/Remote/Repository.cs | 11 ++-- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index bdf0b83..acfa341 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -183,6 +183,11 @@ public async Task Repository_PushAsync() Assert.Equal(index, gotIndex); } + /// + /// Repository_ExistsAsync tests the ExistsAsync method of the Repository + /// + /// + [Fact] public async Task Repository_ExistsAsync() { var blob = @"""hello world"""u8.ToArray(); @@ -199,7 +204,94 @@ public async Task Repository_ExistsAsync() MediaType = OCIMediaTypes.ImageIndex, Size = index.Length }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.NotAcceptable); + } + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var exists = await repo.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await repo.ExistsAsync(indexDesc, cancellationToken); + Assert.True(exists); + } + [Fact] + public async Task Repository_DeleteAsync() + { + var blob = @"""hello world"""u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var blobDeleted = false; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var indexDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + blobDeleted = true; + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + indexDeleted = true; + // no "Docker-Content-Digest" header for manifest deletion + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + await repo.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + await repo.DeleteAsync(indexDesc, cancellationToken); + Assert.True(indexDeleted); } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index b24cc3b..ec4fab2 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -393,8 +393,8 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// private void verifyContentDigest(HttpResponseMessage resp, string expected) { - resp.Headers.TryGetValues(DockerContentDigestHeader, out var digestStr); - if (digestStr != null && !digestStr.Any()) + resp.Content.Headers.TryGetValues(DockerContentDigestHeader, out var digestStr); + if (digestStr == null || !digestStr.Any()) { return; } @@ -403,7 +403,6 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) try { contentDigest = DigestUtil.Parse(digestStr.FirstOrDefault()); - } catch (Exception) { @@ -418,7 +417,6 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in {DockerContentDigestHeader}: received {contentDigest}, while expecting {expected}" ); } - return; } @@ -703,7 +701,7 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO ReferenceObj.VerifyContentDigest(res, refObj.Digest()); // 4. Validate Server Digest (if present) - res.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); + res.Content.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); if (serverHeaderDigest != null) { try @@ -1042,7 +1040,8 @@ public async Task ResolveAsync(string reference, CancellationToken c var refObj = Repo.ParseReference(reference); var refDigest = refObj.Digest(); var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); - var resp = await Repo.Client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); + var resp = await Repo.Client.SendAsync(requestMessage, cancellationToken); return resp.StatusCode switch { HttpStatusCode.OK => GenerateBlobDescriptor(resp, refDigest), From 69ff5806ed48ff33e35c18cf95caf27e121a7698 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 20 May 2023 12:49:38 +0100 Subject: [PATCH 40/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 314 ++++++++++++++++++++++++++++ Oras/Models/Descriptor.cs | 17 +- Oras/Remote/Reference.cs | 3 +- Oras/Remote/Repository.cs | 18 +- 4 files changed, 343 insertions(+), 9 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index acfa341..5674e32 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1,6 +1,7 @@ using Moq; using Moq.Protected; using Oras.Constants; +using Oras.Exceptions; using Oras.Models; using Oras.Remote; using System.Diagnostics; @@ -49,6 +50,7 @@ public async Task Repository_FetchAsync() var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); + resp.RequestMessage = req; if (req.Method != HttpMethod.Get) { Debug.WriteLine("Expected GET request"); @@ -207,6 +209,7 @@ public async Task Repository_ExistsAsync() var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); + res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); @@ -241,6 +244,10 @@ public async Task Repository_ExistsAsync() Assert.True(exists); } + /// + /// Repository_DeleteAsync tests the DeleteAsync method of the Repository + /// + /// [Fact] public async Task Repository_DeleteAsync() { @@ -293,6 +300,313 @@ public async Task Repository_DeleteAsync() await repo.DeleteAsync(indexDesc, cancellationToken); Assert.True(indexDeleted); } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_ResolveAsync() + { + var blob = @"""hello world"""u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) + + { + if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); + // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); + var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); + Assert.Equal(indexDesc, got); + got = await repo.ResolveAsync(reference, cancellationToken); + Assert.Equal(indexDesc, got); + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + got = await repo.ResolveAsync(tagDigestRef, cancellationToken); + Assert.Equal(indexDesc, got); + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await repo.ResolveAsync(fqdnRef, cancellationToken); + Assert.Equal(indexDesc, got); + } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_TagAsync() + { + var blob = """hello"""u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var gotIndex = new byte[indexDesc.Size]; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && + req.RequestUri.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.Found); + } + if (req.Method == HttpMethod.Get && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + return res; + } + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + reference + || req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + + { + if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + gotIndex = req.Content.ReadAsByteArrayAsync().Result; + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAnyAsync( +async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); + await repo.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushReferenceAsync() + { + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var gotIndex = new byte[indexDesc.Size]; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + gotIndex = req.Content.ReadAsByteArrayAsync().Result; + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var streamContent = new MemoryStream(index); + await repo.PushReferenceAsync(indexDesc, streamContent, reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_FetchReferenceAsyc() + { + var blob = """hello"""u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = CalculateDigest(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"""{""manifests"":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = CalculateDigest(index), + MediaType = OCIMediaTypes.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Found); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + + // test with blob digest + await Assert.ThrowsAsync( + async () => await repo.FetchReferenceAsync(blobDesc.Digest, cancellationToken)); + + // test with manifest digest + var data = await repo.FetchReferenceAsync(indexDesc.Digest, cancellationToken); + Assert.Equal(indexDesc, data.Descriptor); + + var buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag + data = await repo.FetchReferenceAsync(reference, cancellationToken); + Assert.Equal(indexDesc, data.Descriptor); + + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag@digest + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + data = await repo.FetchReferenceAsync(tagDigestRef, cancellationToken); + Assert.Equal(indexDesc, data.Descriptor); + + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await repo.FetchReferenceAsync(fqdnRef, cancellationToken); + Assert.Equal(indexDesc, data.Descriptor); + + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + } + + [Fact] + public async Task Repository_TagsAsync() + { + var tagSet = new List>() + { + new(){"the", "quick", "brown", "fox"}, + new(){"jumps", "over", "the", "lazy"}, + new(){"dog"} + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + var q = req.RequestUri.Query; + return new HttpResponseMessage(HttpStatusCode.Found); + }; + } } } + + + diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index 0fce35c..4eb899d 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -1,10 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Text.Json.Serialization; namespace Oras.Models { - public class Descriptor + public class Descriptor : IEquatable { [JsonPropertyName("mediaType")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] @@ -46,6 +47,18 @@ internal static MinimumDescriptor FromOCI(Descriptor descriptor) }; } + public bool Equals(Descriptor other) + { + if (other == null) return false; + return this.MediaType == other.MediaType && this.Digest == other.Digest && this.Size == other.Size; + } + + + + public override int GetHashCode() + { + return HashCode.Combine(MediaType, Digest, Size, URLs, Annotations, Data, Platform, ArtifactType); + } } public class Platform diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 84cbce3..8069981 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -168,9 +168,10 @@ public void ValidateReference() { return; } - if (Reference.IndexOf("@") != -1) + if (Reference.IndexOf(":") != -1) { ValidateReferenceAsDigest(); + return; } else { diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index ec4fab2..7cf363f 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -239,7 +239,7 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation /// /// /// - public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { return await Manifests().FetchReferenceAsync(reference, cancellationToken); } @@ -255,7 +255,7 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, CancellationToken cancellationToken = default) { - await Manifests().FetchReferenceAsync(reference, cancellationToken); + await Manifests().PushReferenceAsync(descriptor, content, reference, cancellationToken); } /// @@ -475,10 +475,10 @@ public ReferenceObj ParseReference(string reference) { Registry = Reference.Registry, Repository = Reference.Repository, - Reference = Reference.Reference + Reference = reference }; //reference is not a FQDN - if (reference.IndexOf("@") is var index && index != 1) + if (reference.IndexOf("@") is var index && index != -1) { // `@` implies *digest*, so drop the *tag* (irrespective of what it is). refObj.Reference = reference[(index + 1)..]; @@ -697,8 +697,14 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO } // 3. Validate Client Reference - var refDigest = refObj.Digest(); - ReferenceObj.VerifyContentDigest(res, refObj.Digest()); + string refDigest = string.Empty; + try + { + refDigest = refObj.Digest(); + } + catch (Exception e) + { + } // 4. Validate Server Digest (if present) res.Content.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); From 92f534f5b22db83f435168747a106338aaeea9a6 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sun, 21 May 2023 00:36:49 +0100 Subject: [PATCH 41/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 71 +++++++++++++++++++++++++++-- Oras/Remote/Repository.cs | 30 +++++++++--- Oras/Remote/ResponseTypes.cs | 6 +-- Oras/Remote/Utils.cs | 20 ++++++-- 4 files changed, 110 insertions(+), 17 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 5674e32..aeabb3b 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -8,6 +8,9 @@ using System.Net; using System.Net.Http.Headers; using System.Text; +using System.Text.Json; +using Oras.Remote; +using System.Text.RegularExpressions; using Xunit; using static Oras.Content.Content; @@ -583,6 +586,12 @@ await Assert.ThrowsAsync( Assert.Equal(index, buf); } + /// + /// Repository_TagsAsync tests the TagsAsync method of the Repository + /// to check if the tags are returned correctly + /// + /// + /// [Fact] public async Task Repository_TagsAsync() { @@ -596,13 +605,69 @@ public async Task Repository_TagsAsync() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method != HttpMethod.Get) + if (req.Method != HttpMethod.Get || + req.RequestUri.AbsolutePath != "/v2/test/tags/list" + ) { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + return new HttpResponseMessage(HttpStatusCode.NotFound); } + var q = req.RequestUri.Query; - return new HttpResponseMessage(HttpStatusCode.Found); + try + { + var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); + if (n != 4) throw new Exception(); + } + catch (Exception e) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + var tags = new List(); + var serverUrl = "http://localhost:5000"; + var matched = Regex.Match(q, @"(?<=test=)\w+").Value; + switch (matched) + { + case "foo": + tags = tagSet[1]; + res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); + break; + case "bar": + tags = tagSet[2]; + break; + default: + tags = tagSet[0]; + res.Headers.Add("Link", $"; rel=\"next\""); + break; + } + var tagObj = new ResponseTypes.Tags() + { + tags = tags.ToArray() + }; + res.Content = new StringContent(JsonSerializer.Serialize(tagObj)); + return res; + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + repo.TagListPageSize = 4; + + var cancellationToken = new CancellationToken(); + + var index = 0; + await repo.TagsAsync("", async (string[] got) => + { + if (index > 2) + { + throw new Exception($"Error out of range: {index}"); + } + + var tags = tagSet[index]; + index++; + Assert.Equal(got, tags); + }, cancellationToken); } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 7cf363f..7064a8d 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -301,7 +301,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken var url = RegistryUtil.BuildRepositoryTagListURL(PlainHTTP, Reference); while (true) { - await tagsAsync(last, fn, url, cancellationToken); + url = await tagsAsync(last, fn, url, cancellationToken); last = ""; } } @@ -322,29 +322,47 @@ public async Task TagsAsync(string last, Action fn, CancellationToken /// private async Task tagsAsync(string last, Action fn, string url, CancellationToken cancellationToken) { + if (PlainHTTP) + { + if (!url.Contains("http")) + { + url = "http://" + url; + } + } + else + { + if (!url.Contains("https")) + { + url = "https://" + url; + } + } + var uri = new UriBuilder(url); + var query = System.Web.HttpUtility.ParseQueryString(uri.Query); if (TagListPageSize > 0 || last != "") { if (TagListPageSize > 0) { - url = url + "?n=" + TagListPageSize; + query["n"] = TagListPageSize.ToString(); + + } if (last != "") { - url = url + "&last=" + last; + query["last"] = last; } } - var resp = await Client.GetAsync(url, cancellationToken); + + uri.Query = query.ToString(); + var resp = await Client.GetAsync(uri.ToString(), cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { throw ErrorUtil.ParseErrorResponse(resp); } - var data = await resp.Content.ReadAsStringAsync(); var page = JsonSerializer.Deserialize(data); fn(page.tags); return Utils.ParseLink(resp); - } /// diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs index 1f943c9..e0b29c7 100644 --- a/Oras/Remote/ResponseTypes.cs +++ b/Oras/Remote/ResponseTypes.cs @@ -1,10 +1,10 @@ namespace Oras.Remote { - internal class ResponseTypes + internal static class ResponseTypes { - public class Tags + public class Tags { - public string[] tags { get; set; } + public string[] tags { get; set; } } } } diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index adf8fb2..771c04b 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -21,12 +21,17 @@ internal class Utils /// public static string ParseLink(HttpResponseMessage resp) { - var link = resp.Headers.GetValues("Link").FirstOrDefault(); - if (String.IsNullOrEmpty(link)) + var link = String.Empty; + if (resp.Headers.TryGetValues("Link", out var values)) + { + link = values.FirstOrDefault(); + } + else { throw new NoLinkHeaderException(); } + if (link[0] != '<') { throw new Exception($"invalid next link {link}: missing '<"); @@ -40,12 +45,17 @@ public static string ParseLink(HttpResponseMessage resp) link = link[1..index]; } - if (!Uri.IsWellFormedUriString(link, UriKind.Absolute)) + if (!Uri.IsWellFormedUriString(link, UriKind.RelativeOrAbsolute)) { - throw new Exception($"invalid next link {link}: not an absolute URL"); + throw new Exception($"invalid next link {link}"); } - return link; + var scheme = resp.RequestMessage.RequestUri.Scheme; + var authority = resp.RequestMessage.RequestUri.Authority; + Uri baseUri = new Uri(scheme+"://"+authority); + Uri resolvedUri = new Uri(baseUri, link); + + return resolvedUri.AbsoluteUri; } /// From af3150b89af69f2eceb2a8acb847bc90b7094fb3 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 22 May 2023 07:04:50 +0100 Subject: [PATCH 42/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 506 ++++++++++++++++++++++++++-- Oras/Remote/Repository.cs | 17 +- 2 files changed, 479 insertions(+), 44 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index aeabb3b..7a3f88b 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -13,6 +13,7 @@ using System.Text.RegularExpressions; using Xunit; using static Oras.Content.Content; +using System.Web; namespace Oras.Tests.RemoteTest { @@ -41,7 +42,7 @@ public async Task Repository_FetchAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() @@ -69,6 +70,7 @@ public async Task Repository_FetchAsync() resp.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return resp; } + if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(OCIMediaTypes.ImageIndex))) @@ -77,12 +79,14 @@ public async Task Repository_FetchAsync() Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } + resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); return resp; } + resp.StatusCode = HttpStatusCode.NotFound; return resp; }; @@ -108,14 +112,14 @@ public async Task Repository_FetchAsync() [Fact] public async Task Repository_PushAsync() { - var blob = @"""hello world"""u8.ToArray(); + var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -139,12 +143,14 @@ public async Task Repository_PushAsync() if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { - if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains("application/octet-stream")) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } + if (!req.RequestUri.Query.Contains("digest=" + blobDesc.Digest)) { resp.StatusCode = HttpStatusCode.BadRequest; @@ -158,6 +164,7 @@ public async Task Repository_PushAsync() return resp; } + if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { @@ -167,12 +174,14 @@ public async Task Repository_PushAsync() resp.StatusCode = HttpStatusCode.BadRequest; return resp; } + var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotIndex); resp.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } + resp.StatusCode = HttpStatusCode.Forbidden; return resp; @@ -195,14 +204,14 @@ public async Task Repository_PushAsync() [Fact] public async Task Repository_ExistsAsync() { - var blob = @"""hello world"""u8.ToArray(); + var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -217,6 +226,7 @@ public async Task Repository_ExistsAsync() { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { res.Content.Headers.Add("Content-Type", "application/octet-stream"); @@ -224,17 +234,21 @@ public async Task Repository_ExistsAsync() res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return res; } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { - if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.NotAcceptable); } + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); return res; } + return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); @@ -254,15 +268,15 @@ public async Task Repository_ExistsAsync() [Fact] public async Task Repository_DeleteAsync() { - var blob = @"""hello world"""u8.ToArray(); + var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; var blobDeleted = false; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -278,6 +292,7 @@ public async Task Repository_DeleteAsync() { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; @@ -285,6 +300,7 @@ public async Task Repository_DeleteAsync() res.StatusCode = HttpStatusCode.Accepted; return res; } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; @@ -292,6 +308,7 @@ public async Task Repository_DeleteAsync() res.StatusCode = HttpStatusCode.Accepted; return res; } + return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); @@ -311,14 +328,14 @@ public async Task Repository_DeleteAsync() [Fact] public async Task Repository_ResolveAsync() { - var blob = @"""hello world"""u8.ToArray(); + var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -335,18 +352,22 @@ public async Task Repository_ResolveAsync() { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) { - if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); @@ -359,7 +380,8 @@ public async Task Repository_ResolveAsync() repo.Client = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); - await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); + await Assert.ThrowsAsync(async () => + await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); Assert.Equal(indexDesc, got); @@ -380,14 +402,14 @@ public async Task Repository_ResolveAsync() [Fact] public async Task Repository_TagAsync() { - var blob = """hello"""u8.ToArray(); + var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -406,9 +428,12 @@ public async Task Repository_TagAsync() { return new HttpResponseMessage(HttpStatusCode.Found); } - if (req.Method == HttpMethod.Get && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + + if (req.Method == HttpMethod.Get && + req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { - if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } @@ -418,27 +443,31 @@ public async Task Repository_TagAsync() res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); return res; } + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + reference || req.RequestUri.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { - if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + gotIndex = req.Content.ReadAsByteArrayAsync().Result; res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } + return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository("localhost:5000/test"); repo.Client = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); - await Assert.ThrowsAnyAsync( -async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); + await Assert.ThrowsAnyAsync( + async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); await repo.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); @@ -452,7 +481,7 @@ await Assert.ThrowsAnyAsync( [Fact] public async Task Repository_PushReferenceAsync() { - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -468,15 +497,18 @@ public async Task Repository_PushReferenceAsync() res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/manifests/" + reference) { - if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(OCIMediaTypes.ImageIndex)) + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(OCIMediaTypes.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + gotIndex = req.Content.ReadAsByteArrayAsync().Result; res.Content.Headers.Add("Docker-Content-Digest", indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } + return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository("localhost:5000/test"); @@ -495,14 +527,14 @@ public async Task Repository_PushReferenceAsync() [Fact] public async Task Repository_FetchReferenceAsyc() { - var blob = """hello"""u8.ToArray(); + var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint)blob.Length + Size = (uint) blob.Length }; - var index = @"""{""manifests"":[]}"""u8.ToArray(); + var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = CalculateDigest(index), @@ -548,7 +580,7 @@ public async Task Repository_FetchReferenceAsyc() var cancellationToken = new CancellationToken(); // test with blob digest - await Assert.ThrowsAsync( + await Assert.ThrowsAsync( async () => await repo.FetchReferenceAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest @@ -597,9 +629,9 @@ public async Task Repository_TagsAsync() { var tagSet = new List>() { - new(){"the", "quick", "brown", "fox"}, - new(){"jumps", "over", "the", "lazy"}, - new(){"dog"} + new() {"the", "quick", "brown", "fox"}, + new() {"jumps", "over", "the", "lazy"}, + new() {"dog"} }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { @@ -607,7 +639,7 @@ public async Task Repository_TagsAsync() res.RequestMessage = req; if (req.Method != HttpMethod.Get || req.RequestUri.AbsolutePath != "/v2/test/tags/list" - ) + ) { return new HttpResponseMessage(HttpStatusCode.NotFound); } @@ -640,6 +672,7 @@ public async Task Repository_TagsAsync() res.Headers.Add("Link", $"; rel=\"next\""); break; } + var tagObj = new ResponseTypes.Tags() { tags = tags.ToArray() @@ -669,9 +702,410 @@ await repo.TagsAsync("", async (string[] got) => Assert.Equal(got, tags); }, cancellationToken); } + + /// + /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + } + + /// + /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_CanSeek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + + IEnumerable rangeHeader; + if (req.Headers.TryGetValues("Range", out rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + + long start = 0, end = 0; + try + { + start = req.Headers.Range.Ranges.First().From.Value; + end = req.Headers.Range.Ranges.First().To.Value; + } + catch (Exception e) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + if (start < 0 || start > end || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + end++; + if (end > blobDesc.Size) + { + end = blobDesc.Size; + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int) start..(int) end]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + seekable = true; + stream = await store.FetchAsync(blobDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + buf = new byte[stream.Length - 3]; + stream.Seek(3, SeekOrigin.Begin); + await stream.ReadAsync(buf, cancellationToken); + var seg = blob[3..]; + Assert.Equal(seg, buf); + } + + /// + /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_ZeroSizedBlob() + { + var blob = ""u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (req.Headers.TryGetValues("Range", out var rangeHeader)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + } + + /// + /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_PushAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var gotBlob = new byte[blob.Length]; + var uuid = Guid.NewGuid().ToString(); + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri.AbsolutePath == $"/v2/test/blobs/uploads/") + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Add("Location", "/v2/test/blobs/uploads/" + uuid); + return res; + } + + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + var buf = new byte[req.Content.Headers.ContentLength.Value]; + // read content into buffer + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + res.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + } + + /// + /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_ExistsAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(content), + Size = content.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var exists = await store.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exists); + } + + /// + /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_DeleteAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var blobDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + blobDeleted = true; + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(content), + Size = content.Length + }; + Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + [Fact] + public async Task BlobStore_ResolveAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + Assert.Equal(blobDesc.Size,got.Size); + } } } - diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 7064a8d..404250a 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -1006,21 +1006,22 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok } // monolithic upload - var location = resp.Headers.Location; + var location = resp.RequestMessage.RequestUri.Scheme+"://"+ resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; // work-around solution for https://github.com/oras-project/oras-go/issues/177 - // For some registries, if the port 443 is explicitly set to the hostname + // For some registries, if the port 443 is explicitly set to the hostname plicitly set to the hostname // like registry.wabbit-networks.io:443/myrepo, blob push will fail since // the hostname of the Location header in the response is set to // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. - - var locationHostname = location.Host; - var locationPort = location.Port; + var uri = new Uri(location); + var locationHostname = uri.Host ; + var locationPort = uri.Port; // if location port 443 is missing, add it back if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) { - location = new Uri($"{locationHostname}:{reqPort}"); + location = new Uri($"{locationHostname}:{reqPort}").ToString(); } - url = location.ToString(); + + url = location; var req = new HttpRequestMessage(HttpMethod.Put, url); req.Content = new StreamContent(content); @@ -1034,7 +1035,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok req.Content.Headers.Add("Content-Type", "application/octet-stream"); // add digest key to query string with expected digest value - var query = HttpUtility.ParseQueryString(location.Query); + var query = HttpUtility.ParseQueryString(new Uri(location).Query); query.Add("digest", expected.Digest); req.RequestUri = new Uri($"{req.RequestUri}?digest={expected.Digest}"); From e2ede17abed12947db23e5afb819d9482028c967 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 22 May 2023 07:41:43 +0100 Subject: [PATCH 43/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 84 ++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 7a3f88b..5e54294 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1064,6 +1064,10 @@ public async Task BlobStore_DeleteAsync() Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } + /// + /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. + /// + /// [Fact] public async Task BlobStore_ResolveAsync() { @@ -1094,7 +1098,6 @@ public async Task BlobStore_ResolveAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; - var repo = new Repository("localhost:5000/test"); repo.Client = CustomClient(func); repo.PlainHTTP = true; @@ -1103,6 +1106,85 @@ public async Task BlobStore_ResolveAsync() var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); Assert.Equal(blobDesc.Size,got.Size); + + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + } + + /// + /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + + // test with digest + var gotDesc = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var buf = new byte[gotDesc.Descriptor.Size]; + await gotDesc.Stream.ReadAsync(buf,cancellationToken); + Assert.Equal(blob, buf); + + // test with FQDN reference + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + gotDesc = await store.FetchReferenceAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(content), + Size = content.Length + }; + // test with other digest + await Assert.ThrowsAsync(async () => await store.FetchReferenceAsync(contentDesc.Digest, cancellationToken)); } } } From 355379902c95c847f69cd1ea259c99b39250931a Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Mon, 22 May 2023 16:12:41 +0100 Subject: [PATCH 44/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 319 +++++++++++++++++++++++++++- Oras/Remote/ErrorUtil.cs | 8 +- Oras/Remote/ManifestUtil.cs | 2 +- Oras/Remote/Reference.cs | 2 +- Oras/Remote/Repository.cs | 99 +++++---- 5 files changed, 372 insertions(+), 58 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 5e54294..90758b3 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1,4 +1,5 @@ -using Moq; +using System.Collections.Immutable; +using Moq; using Moq.Protected; using Oras.Constants; using Oras.Exceptions; @@ -14,11 +15,102 @@ using Xunit; using static Oras.Content.Content; using System.Web; +using Xunit.Abstractions; namespace Oras.Tests.RemoteTest { public class RemoteTest { + public struct TestIOStruct + { + public bool isTag; + public bool errExpectedOnHEAD; + public string serverCalculatedDigest; + public string clientSuppliedReference; + public bool errExpectedOnGET; + } + + private byte[] theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); + private const string theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; + + private const string dockerContentDigestHeader = "Docker-Content-Digest"; + // The following truth table aims to cover the expected GET/HEAD request outcome + // for all possible permutations of the client/server "containing a digest", for + // both Manifests and Blobs. Where the results between the two differ, the index + // of the first column has an exclamation mark. + // + // The client is said to "contain a digest" if the user-supplied reference string + // is of the form that contains a digest rather than a tag. The server, on the + // other hand, is said to "contain a digest" if the server responded with the + // special header `Docker-Content-Digest`. + // + // In this table, anything denoted with an asterisk indicates that the true + // response should actually be the opposite of what's expected; for example, + // `*PASS` means we will get a `PASS`, even though the true answer would be its + // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. + // This function has blind-spots, and while it can expend power to gain sight, + // i.e., perform the expensive validation, we chose not to. The reason is two- + // fold: a) we "know" that even if we say "!PASS", it will eventually fail later + // when checks are performed, and with that assumption, we have the luxury for + // the second point, which is b) performance. + // + // _______________________________________________________________________________________________________________ + // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | + // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ + // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | + // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | + // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | + // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | + // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | + // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | + // --------------------------------------------------------------------------------------------------------------- + + /// + /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different + /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for + /// both Manifests and Blobs. + /// + /// + public static Dictionary GetTestIOStructMapForGetDescriptorClass() + { + string correctDigest = $"sha256:{theAmazingBanDigest}"; + string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + + return new Dictionary + { + ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct + { + isTag = true, + errExpectedOnHEAD = true + }, + ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct + { + isTag = true, + serverCalculatedDigest = correctDigest + }, + ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + isTag = true, + serverCalculatedDigest = incorrectDigest + }, + ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct + { + clientSuppliedReference = correctDigest + }, + ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct + { + clientSuppliedReference = correctDigest, + serverCalculatedDigest = correctDigest + }, + ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + clientSuppliedReference = correctDigest, + serverCalculatedDigest = incorrectDigest, + errExpectedOnHEAD = true, + errExpectedOnGET = true + } + }; + } public static HttpClient CustomClient(Func func) { var moqHandler = new Mock(); @@ -932,7 +1024,8 @@ public async Task BlobStore_PushAsync() if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { - if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") + if (req.Headers.TryGetValues("Content-Type", out var contentType) && + contentType.FirstOrDefault() != "application/octet-stream") { return new HttpResponseMessage(HttpStatusCode.BadRequest); } @@ -941,6 +1034,7 @@ public async Task BlobStore_PushAsync() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + var buf = new byte[req.Content.Headers.ContentLength.Value]; // read content into buffer var stream = req.Content!.ReadAsStream(cancellationToken); @@ -986,11 +1080,12 @@ public async Task BlobStore_ExistsAsync() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method != HttpMethod.Head) + if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); @@ -1036,6 +1131,7 @@ public async Task BlobStore_DeleteAsync() res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { blobDeleted = true; @@ -1088,6 +1184,7 @@ public async Task BlobStore_ResolveAsync() res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); @@ -1105,7 +1202,7 @@ public async Task BlobStore_ResolveAsync() var store = new BlobStore(repo); var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); - Assert.Equal(blobDesc.Size,got.Size); + Assert.Equal(blobDesc.Size, got.Size); var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; got = await store.ResolveAsync(fqdnRef, cancellationToken); @@ -1118,7 +1215,8 @@ public async Task BlobStore_ResolveAsync() Digest = CalculateDigest(content), Size = content.Length }; - await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + await Assert.ThrowsAsync(async () => + await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// @@ -1145,6 +1243,7 @@ public async Task BlobStore_FetchReferenceAsync() res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); @@ -1152,6 +1251,7 @@ public async Task BlobStore_FetchReferenceAsync() res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return res; } + return new HttpResponseMessage(HttpStatusCode.NotFound); }; @@ -1167,7 +1267,7 @@ public async Task BlobStore_FetchReferenceAsync() Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var buf = new byte[gotDesc.Descriptor.Size]; - await gotDesc.Stream.ReadAsync(buf,cancellationToken); + await gotDesc.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test with FQDN reference @@ -1184,10 +1284,211 @@ public async Task BlobStore_FetchReferenceAsync() Size = content.Length }; // test with other digest - await Assert.ThrowsAsync(async () => await store.FetchReferenceAsync(contentDesc.Digest, cancellationToken)); + await Assert.ThrowsAsync(async () => + await store.FetchReferenceAsync(contentDesc.Digest, cancellationToken)); } - } -} + /// + /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync_Seek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + IEnumerable rangeHeader; + if (req.Headers.TryGetValues("Range", out rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + + long start = 0; + try + { + start = req.Headers.Range.Ranges.First().From.Value; + } + catch (Exception e) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + if (start < 0 || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int) start..]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + + var store = new BlobStore(repo); + + // test non-seekable content + + var data = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + var buf = new byte[data.Descriptor.Size]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + // test seekable content + seekable = true; + data = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + data.Stream.Seek(3, SeekOrigin.Begin); + buf = new byte[data.Descriptor.Size - 3]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob[3..], buf); + } + + + /// + /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. + /// + /// + /// + [Fact] + public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() + { + var reference = new ReferenceObj() + { + Registry = "eastern.haan.com", + Reference = "", + Repository = "from25to220ce" + }; + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + if (dcdIOStruct.isTag) + { + continue; + } + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.Reference = dcdIOStruct.clientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add(dockerContentDigestHeader, new string[] { dcdIOStruct.serverCalculatedDigest }); + } + if (!resp.Content.Headers.TryGetValues(dockerContentDigestHeader, out IEnumerable values)) + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add(dockerContentDigestHeader, new string[] { dcdIOStruct.serverCalculatedDigest }); + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + } + else + { + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + } + + var d = string.Empty; + try + { + d = reference.Digest(); + } + catch (Exception e) + { + throw new Exception( + $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); + } + + var errExpected = new bool[] { dcdIOStruct.errExpectedOnGET, dcdIOStruct.errExpectedOnHEAD }[i]; + if (d.Length == 0) + { + // To avoid an otherwise impossible scenario in the tested code + // path, we set d so that verifyContentDigest does not break. + d = dcdIOStruct.serverCalculatedDigest; + } + + var err = false; + try + { + Repository.GenerateBlobDescriptor(resp, d); + + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + + if (errExpected && !err) + { + throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); + } + } + } + + } + + + } +} \ No newline at end of file diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs index 1919d30..5e1625d 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtil.cs @@ -1,5 +1,6 @@ using System; using System.Net.Http; +using System.Threading.Tasks; namespace Oras.Remote { @@ -10,14 +11,15 @@ internal class ErrorUtil /// /// /// - public static Exception ParseErrorResponse(HttpResponseMessage resp) + public static async Task ParseErrorResponse(HttpResponseMessage resp) { - return new Exception(new + var body = await resp.Content.ReadAsStringAsync(); + return new Exception( new { resp.RequestMessage.Method, URL = resp.RequestMessage.RequestUri, resp.StatusCode, - Errors = resp.Content.ReadAsStringAsync().Result + Errors = body }.ToString()); } } diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index 09ec95a..c133d26 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -50,4 +50,4 @@ public static string ManifestAcceptHeader(string[] manifestMediaTypes) return string.Join(",", manifestMediaTypes); } } -} \ No newline at end of file +} diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 8069981..bfc3d30 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -49,7 +49,7 @@ public class ReferenceObj /// /// digestRegexp checks the digest. /// - public static string digestRegexp = @"^sha256:[0-9a-fA-F]{64}$"; + public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; public ReferenceObj ParseReference(string artifact) { var parts = artifact.Split("/", 2); diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 404250a..2e05520 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -154,7 +154,7 @@ public async Task PingAsync(CancellationToken cancellationToken) case HttpStatusCode.NotFound: throw new NotFoundException($"Repository {Reference} not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } } /// @@ -356,7 +356,7 @@ private async Task tagsAsync(string last, Action fn, string ur var resp = await Client.GetAsync(uri.ToString(), cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } var data = await resp.Content.ReadAsStringAsync(); @@ -396,7 +396,7 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } } @@ -512,6 +512,36 @@ public ReferenceObj ParseReference(string reference) } + /// + /// GenerateBlobDescriptor returns a descriptor generated from the response. + /// + /// + /// + /// + /// + public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string refDigest) + { + var mediaType = resp.Content.Headers.ContentType.MediaType; + if (string.IsNullOrEmpty(mediaType)) + { + mediaType = "application/octet-stream"; + } + var size = resp.Content.Headers.ContentLength!.Value; + if (size == -1) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); + } + + ReferenceObj.VerifyContentDigest(resp, refDigest); + + return new Descriptor + { + MediaType = mediaType, + Digest = refDigest, + Size = size + }; + } + } public class ManifestStore : IManifestStore @@ -547,7 +577,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } var mediaType = resp.Content.Headers?.ContentType.MediaType; if (mediaType != target.MediaType) @@ -642,7 +672,7 @@ private async Task pushAsync(Descriptor expected, Stream stream, string referenc var resp = await client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } ReferenceObj.VerifyContentDigest(resp, expected.Digest); } @@ -681,7 +711,7 @@ public async Task ResolveAsync(string reference, CancellationToken c { HttpStatusCode.OK => generateDescriptor(res, refObj, req.Method), HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), - _ => throw ErrorUtil.ParseErrorResponse(res) + _ => throw await ErrorUtil.ParseErrorResponse(res) }; } @@ -858,7 +888,7 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken case HttpStatusCode.NotFound: throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } } @@ -954,7 +984,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } } @@ -1002,11 +1032,19 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok var reqPort = resp.RequestMessage.RequestUri.Port; if (resp.StatusCode != HttpStatusCode.Accepted) { - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } + string location = String.Empty; // monolithic upload - var location = resp.RequestMessage.RequestUri.Scheme+"://"+ resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; + if (!resp.Headers.Location.IsAbsoluteUri) + { + location = resp.RequestMessage.RequestUri.Scheme+"://"+ resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; + } + else + { + location = resp.Headers.Location.ToString(); + } // work-around solution for https://github.com/oras-project/oras-go/issues/177 // For some registries, if the port 443 is explicitly set to the hostname plicitly set to the hostname // like registry.wabbit-networks.io:443/myrepo, blob push will fail since @@ -1048,7 +1086,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok resp = await Repo.Client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } return; @@ -1069,9 +1107,9 @@ public async Task ResolveAsync(string reference, CancellationToken c var resp = await Repo.Client.SendAsync(requestMessage, cancellationToken); return resp.StatusCode switch { - HttpStatusCode.OK => GenerateBlobDescriptor(resp, refDigest), + HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), HttpStatusCode.NotFound => throw new NotFoundException($"{refObj.Reference}: not found"), - _ => throw ErrorUtil.ParseErrorResponse(resp) + _ => throw await ErrorUtil.ParseErrorResponse(resp) }; } @@ -1110,7 +1148,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT } else { - desc = GenerateBlobDescriptor(resp, refDigest); + desc = Repository.GenerateBlobDescriptor(resp, refDigest); } // check server range request capability. // Docker spec allows range header form of "Range: bytes=-". @@ -1145,39 +1183,12 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT case HttpStatusCode.NotFound: throw new NotFoundException(); default: - throw ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtil.ParseErrorResponse(resp); } } - /// - /// GenerateBlobDescriptor returns a descriptor generated from the response. - /// - /// - /// - /// - /// - private Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string refDigest) - { - var mediaType = resp.Content.Headers.ContentType.MediaType; - if (string.IsNullOrEmpty(mediaType)) - { - mediaType = "application/octet-stream"; - } - var size = resp.Content.Headers.ContentLength!.Value; - if (size == -1) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); - } - - ReferenceObj.VerifyContentDigest(resp, refDigest); - - return new Descriptor - { - MediaType = mediaType, - Digest = refDigest, - Size = size - }; - } + } + } From 4fdc00028af4447ffa0fba8a5ec09806dc3f4519 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 24 May 2023 07:25:09 +0100 Subject: [PATCH 45/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 168 +++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 2 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 90758b3..548aec3 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -16,6 +16,7 @@ using static Oras.Content.Content; using System.Web; using Xunit.Abstractions; +using System; namespace Oras.Tests.RemoteTest { @@ -1488,7 +1489,170 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() } } - - + + + /// + /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Content.Headers.Add("Content-Type", new string[] { OCIMediaTypes.ImageManifest }); + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var data = await store.FetchAsync(manifestDesc, cancellationToken); + var buf = new byte[data.Length]; + await data.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(content), + Size = content.Length + }; + Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); + } + + /// + /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + byte[] gotManifest = null; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + var buf = new byte[req.Content.Headers.ContentLength.Value]; + req.Content.ReadAsByteArrayAsync().Result.CopyTo(buf, 0); + gotManifest = buf; + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + else + { + return new HttpResponseMessage(HttpStatusCode.Forbidden); + } + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); + Assert.Equal(manifest, gotManifest); + } + + /// + /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ExistAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { OCIMediaTypes.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var exist = await store.ExistsAsync(manifestDesc, cancellationToken); + Assert.True(exist); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(content), + Size = content.Length + }; + exist = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exist); + } + + [Fact] + public async Task ManifestStore_DeleteAsync() + { + var + } } } \ No newline at end of file From 511ad1686fb9213f5305ead48db1ae375ff1c637 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Wed, 24 May 2023 15:17:48 +0100 Subject: [PATCH 46/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 389 +++++++++++++++++++++++++++- Oras/Remote/DigestUtil.cs | 6 +- Oras/Remote/Repository.cs | 19 +- 3 files changed, 400 insertions(+), 14 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 548aec3..74e21a4 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1487,7 +1487,6 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() } } } - } @@ -1649,10 +1648,396 @@ public async Task ManifestStore_ExistAsync() Assert.False(exist); } + /// + /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. + /// + /// [Fact] public async Task ManifestStore_DeleteAsync() { - var + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + var manifestDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { OCIMediaTypes.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.True(manifestDeleted); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(content), + Size = content.Length + }; + Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + /// + /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ResolveAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { OCIMediaTypes.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); + Assert.Equal(manifestDesc, got); + got = await store.ResolveAsync(reference, cancellationToken); + Assert.Equal(manifestDesc, got); + + var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; + got = await store.ResolveAsync(tagDigestRef, cancellationToken); + Assert.Equal(manifestDesc, got); + + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.Equal(manifestDesc, got); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(content), + Size = content.Length + }; + Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + + } + + /// + /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchReferenceAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageManifest, + Digest = CalculateDigest(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Content.Headers.Add("Docker-Content-Digest", new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { OCIMediaTypes.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // test with tag + var data = await store.FetchReferenceAsync(reference, cancellationToken); + Assert.Equal(manifestDesc, data.Descriptor); + + var buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with other tag + var randomRef = "whatever"; + await Assert.ThrowsAsync(async () => await store.FetchReferenceAsync(randomRef, cancellationToken)); + + // test with digest + data = await store.FetchReferenceAsync(manifestDesc.Digest, cancellationToken); + Assert.Equal(manifestDesc, data.Descriptor); + + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with tag@digest + var tagDigestRef = randomRef + "@" + manifestDesc.Digest; + data = await store.FetchReferenceAsync(tagDigestRef, cancellationToken); + Assert.Equal(manifestDesc, data.Descriptor); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await store.FetchReferenceAsync(fqdnRef, cancellationToken); + Assert.Equal(manifestDesc, data.Descriptor); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + } + + /// + /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_TagAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor + { + MediaType = "test", + Digest = CalculateDigest(blob), + Size = blob.Length + }; + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") + { + res.StatusCode = HttpStatusCode.NotFound; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(indexDesc.MediaType)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Docker-Content-Digest", new string[] { indexDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); + return res; + } + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + var buf = new byte[req.Content.Headers.ContentLength.Value]; + req.Content.ReadAsByteArrayAsync().Result.CopyTo(buf, 0); + gotIndex = buf; + res.Content.Headers.Add("Docker-Content-Digest", new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); + + await store.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + + gotIndex = null; + await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushReferenceAsync() + { + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor + { + MediaType = OCIMediaTypes.ImageIndex, + Digest = CalculateDigest(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + + if (req.Method == HttpMethod.Put && req.RequestUri.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + var buf = new byte[req.Content.Headers.ContentLength.Value]; + req.Content.ReadAsByteArrayAsync().Result.CopyTo(buf, 0); + gotIndex = buf; + res.Content.Headers.Add("Docker-Content-Digest", new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + var repo = new Repository("localhost:5000/test"); + repo.Client = CustomClient(func); + repo.PlainHTTP = true; + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushReferenceAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + + public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() + { + var reference = new ReferenceObj() + { + Registry = "eastern.haan.com", + Reference = "", + Repository = "from25to220ce" + }; + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + var repo = new Repository(reference.Repository+"/"+reference.Repository); + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + var s = new ManifestStore(repo); + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.Reference = dcdIOStruct.clientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add(dockerContentDigestHeader, new string[] { dcdIOStruct.serverCalculatedDigest }); + } + else + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add(dockerContentDigestHeader, new string[] { dcdIOStruct.serverCalculatedDigest }); + } + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + var errExpected = new bool[] { dcdIOStruct.errExpectedOnGET, dcdIOStruct.errExpectedOnHEAD }[i]; + + var err = false; + try + { + s.GenerateDescriptor(resp, reference,method); + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + if (errExpected && !err) + { + throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); + } + } + } + } } } \ No newline at end of file diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Remote/DigestUtil.cs index 7f7dc6a..8afc421 100644 --- a/Oras/Remote/DigestUtil.cs +++ b/Oras/Remote/DigestUtil.cs @@ -23,14 +23,14 @@ public static string Parse(string digest) } /// - /// FromBytes generates a digest from the content. + /// FromBytes generates a digest from a byte. /// /// /// - public static string FromBytes(HttpContent content) + public static string FromBytes(byte[] content) { using var sha256 = SHA256.Create(); - var hash = sha256.ComputeHash(content.ReadAsByteArrayAsync().Result); + var hash = sha256.ComputeHash(content); var digest = $"sha256:{Convert.ToBase64String(hash)}"; return digest; } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 2e05520..bbb6db2 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -709,7 +709,7 @@ public async Task ResolveAsync(string reference, CancellationToken c return res.StatusCode switch { - HttpStatusCode.OK => generateDescriptor(res, refObj, req.Method), + HttpStatusCode.OK => GenerateDescriptor(res, refObj, req.Method), HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), _ => throw await ErrorUtil.ParseErrorResponse(res) }; @@ -723,14 +723,14 @@ public async Task ResolveAsync(string reference, CancellationToken c /// /// /// - private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refObj, HttpMethod httpMethod) + public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refObj, HttpMethod httpMethod) { string mediaType; try { // 1. Validate Content-Type mediaType = res.Content.Headers.ContentType.MediaType; - MediaTypeHeaderValue.TryParse(mediaType.ToString(), out var parsedMediaType); + MediaTypeHeaderValue.Parse(mediaType); } catch (Exception e) @@ -756,7 +756,7 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO // 4. Validate Server Digest (if present) res.Content.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); - if (serverHeaderDigest != null) + if (!string.IsNullOrEmpty(serverHeaderDigest.First())) { try { @@ -771,7 +771,7 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO // 5. Now, look for specific error conditions; string contentDigest; - if (serverHeaderDigest.FirstOrDefault().Length == 0) + if (serverHeaderDigest.First().Length == 0) { if (httpMethod == HttpMethod.Head) { @@ -826,15 +826,16 @@ private Descriptor generateDescriptor(HttpResponseMessage res, ReferenceObj refO /// private string calculateDigestFromResponse(HttpResponseMessage res, long maxMetadataBytes) { + byte[] content = null; try { - byte[] content = Utils.LimitReader(res.Content, maxMetadataBytes); + content = Utils.LimitReader(res.Content, maxMetadataBytes); } catch (Exception ex) { throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {ex.Message}"); } - return DigestUtil.FromBytes(res.Content); + return DigestUtil.FromBytes(content); } /// @@ -865,7 +866,7 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken /// /// /// - public async Task<(Descriptor, Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) + public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { var refObj = Repo.ParseReference(reference); var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); @@ -882,7 +883,7 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken } else { - desc = generateDescriptor(resp, refObj, HttpMethod.Get); + desc = GenerateDescriptor(resp, refObj, HttpMethod.Get); } return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: From 6a2fe7d1cf53bd458f43c09860764f576e978657 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 04:24:02 +0100 Subject: [PATCH 47/77] removed see Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 63 ++-- Oras/Exceptions/NoLinkHeaderException.cs | 24 -- Oras/Interfaces/Registry/IRepositoryOption.cs | 4 +- Oras/Oras.csproj | 3 +- Oras/Remote/ErrorUtil.cs | 12 +- Oras/Remote/ManifestUtil.cs | 4 +- Oras/Remote/Reference.cs | 24 +- Oras/Remote/Registry.cs | 64 ++++ Oras/Remote/Repository.cs | 306 +++++++----------- Oras/Remote/ResponseTypes.cs | 10 - .../{RegistryUtil.cs => URLUtiliity.cs} | 46 +-- Oras/Remote/Utils.cs | 22 +- .../DigestUtil.cs => Utils/DigestUtility.cs} | 7 +- 13 files changed, 282 insertions(+), 307 deletions(-) delete mode 100644 Oras/Exceptions/NoLinkHeaderException.cs create mode 100644 Oras/Remote/Registry.cs delete mode 100644 Oras/Remote/ResponseTypes.cs rename Oras/Remote/{RegistryUtil.cs => URLUtiliity.cs} (64%) rename Oras/{Remote/DigestUtil.cs => Utils/DigestUtility.cs} (87%) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 74e21a4..61bfec7 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -184,7 +184,7 @@ public async Task Repository_FetchAsync() return resp; }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc, cancellationToken); @@ -281,7 +281,7 @@ public async Task Repository_PushAsync() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); @@ -345,7 +345,7 @@ public async Task Repository_ExistsAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var exists = await repo.ExistsAsync(blobDesc, cancellationToken); @@ -405,7 +405,7 @@ public async Task Repository_DeleteAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); await repo.DeleteAsync(blobDesc, cancellationToken); @@ -470,7 +470,7 @@ public async Task Repository_ResolveAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); await Assert.ThrowsAsync(async () => @@ -556,7 +556,7 @@ public async Task Repository_TagAsync() return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); await Assert.ThrowsAnyAsync( @@ -605,7 +605,7 @@ public async Task Repository_PushReferenceAsync() return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); @@ -668,7 +668,7 @@ public async Task Repository_FetchReferenceAsyc() return new HttpResponseMessage(HttpStatusCode.Found); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); @@ -765,18 +765,13 @@ public async Task Repository_TagsAsync() res.Headers.Add("Link", $"; rel=\"next\""); break; } - - var tagObj = new ResponseTypes.Tags() - { - tags = tags.ToArray() - }; - res.Content = new StringContent(JsonSerializer.Serialize(tagObj)); + res.Content = new StringContent(JsonSerializer.Serialize(tags)); return res; }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; repo.TagListPageSize = 4; @@ -831,7 +826,7 @@ public async Task BlobStore_FetchAsync() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -925,7 +920,7 @@ public async Task BlobStore_FetchAsync_CanSeek() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -986,7 +981,7 @@ public async Task BlobStore_FetchAsync_ZeroSizedBlob() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1048,7 +1043,7 @@ public async Task BlobStore_PushAsync() return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1098,7 +1093,7 @@ public async Task BlobStore_ExistsAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1144,7 +1139,7 @@ public async Task BlobStore_DeleteAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1197,7 +1192,7 @@ public async Task BlobStore_ResolveAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1257,7 +1252,7 @@ public async Task BlobStore_FetchReferenceAsync() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); @@ -1365,7 +1360,7 @@ public async Task BlobStore_FetchReferenceAsync_Seek() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); @@ -1403,7 +1398,7 @@ public async Task BlobStore_FetchReferenceAsync_Seek() [Fact] public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { - var reference = new ReferenceObj() + var reference = new RemoteReference() { Registry = "eastern.haan.com", Reference = "", @@ -1527,7 +1522,7 @@ public async Task ManifestStore_FetchAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1586,7 +1581,7 @@ public async Task ManifestStore_PushAsync() } }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1630,7 +1625,7 @@ public async Task ManifestStore_ExistAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1691,7 +1686,7 @@ public async Task ManifestStore_DeleteAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1745,7 +1740,7 @@ public async Task ManifestStore_ResolveAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1810,7 +1805,7 @@ public async Task ManifestStore_FetchReferenceAsync() return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1916,7 +1911,7 @@ public async Task ManifestStore_TagAsync() }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1971,7 +1966,7 @@ public async Task ManifestStore_PushReferenceAsync() return res; }; var repo = new Repository("localhost:5000/test"); - repo.Client = CustomClient(func); + repo.HttpClient = CustomClient(func); repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); @@ -1982,7 +1977,7 @@ public async Task ManifestStore_PushReferenceAsync() public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { - var reference = new ReferenceObj() + var reference = new RemoteReference() { Registry = "eastern.haan.com", Reference = "", diff --git a/Oras/Exceptions/NoLinkHeaderException.cs b/Oras/Exceptions/NoLinkHeaderException.cs deleted file mode 100644 index 438d01c..0000000 --- a/Oras/Exceptions/NoLinkHeaderException.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; - -namespace Oras.Exceptions -{ - /// - /// NoLinkHeaderException is thrown when a link header is missing. - /// - public class NoLinkHeaderException : Exception - { - public NoLinkHeaderException() - { - } - - public NoLinkHeaderException(string message) - : base(message) - { - } - - public NoLinkHeaderException(string message, Exception inner) - : base(message, inner) - { - } - } -} diff --git a/Oras/Interfaces/Registry/IRepositoryOption.cs b/Oras/Interfaces/Registry/IRepositoryOption.cs index f1f96d7..79bc1ce 100644 --- a/Oras/Interfaces/Registry/IRepositoryOption.cs +++ b/Oras/Interfaces/Registry/IRepositoryOption.cs @@ -11,12 +11,12 @@ public interface IRepositoryOption /// /// Client is the underlying HTTP client used to access the remote registry. /// - public HttpClient Client { get; set; } + public HttpClient HttpClient { get; set; } /// /// Reference references the remote repository. /// - public ReferenceObj Reference { get; set; } + public RemoteReference RemoteReference { get; set; } /// /// PlainHTTP signals the transport to access the remote repository via HTTP diff --git a/Oras/Oras.csproj b/Oras/Oras.csproj index d588c19..e1639fc 100644 --- a/Oras/Oras.csproj +++ b/Oras/Oras.csproj @@ -10,4 +10,5 @@ - \ No newline at end of file + + diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs index 5e1625d..c94d853 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtil.cs @@ -9,16 +9,16 @@ internal class ErrorUtil /// /// ParseErrorResponse parses the error returned by the remote registry. /// - /// + /// /// - public static async Task ParseErrorResponse(HttpResponseMessage resp) + internal static async Task ParseErrorResponse(HttpResponseMessage response) { - var body = await resp.Content.ReadAsStringAsync(); + var body = await response.Content.ReadAsStringAsync(); return new Exception( new { - resp.RequestMessage.Method, - URL = resp.RequestMessage.RequestUri, - resp.StatusCode, + response.RequestMessage.Method, + URL = response.RequestMessage.RequestUri, + response.StatusCode, Errors = body }.ToString()); } diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index c133d26..7b51899 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -4,9 +4,9 @@ namespace Oras.Remote { - static class ManifestUtil + internal static class ManifestUtil { - public static string[] DefaultManifestMediaTypes = new[] + internal static string[] DefaultManifestMediaTypes = new[] { DockerMediaTypes.Manifest, DockerMediaTypes.ManifestList, diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index bfc3d30..d2e2055 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -6,7 +6,7 @@ namespace Oras.Remote { - public class ReferenceObj + public class RemoteReference { /// /// Registry is the name of the registry. It is usually the domain name of the registry optionally with a port. @@ -50,9 +50,9 @@ public class ReferenceObj /// digestRegexp checks the digest. /// public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; - public ReferenceObj ParseReference(string artifact) + public RemoteReference ParseReference(string artifact) { - var parts = artifact.Split("/", 2); + var parts = artifact.Split('/', 2); if (parts.Length == 1) { throw new InvalidReferenceException($"missing repository"); @@ -62,21 +62,21 @@ public ReferenceObj ParseReference(string artifact) string repository = String.Empty; string reference = String.Empty; - if (path.IndexOf("@") is var index && index != -1) + if (path.IndexOf('@') is var index && index != -1) { // digest found; Valid From A (if not B) isTag = false; repository = path[..index]; reference = path[(index + 1)..]; - if (repository.IndexOf(":") is var indexOfColon && indexOfColon != -1) + if (repository.IndexOf(':') is var indexOfColon && indexOfColon != -1) { // tag found ( and now dropped without validation ) since the // digest already present; Valid Form B repository = repository[..indexOfColon]; } } - else if (path.IndexOf(":") is var indexOfColon && indexOfColon != -1) + else if (path.IndexOf(':') is var indexOfColon && indexOfColon != -1) { // tag found; Valid Form C isTag = true; @@ -88,7 +88,7 @@ public ReferenceObj ParseReference(string artifact) // empty `reference`; Valid Form D repository = path; } - var refObj = new ReferenceObj + var refObj = new RemoteReference { Registry = registry, Repository = repository, @@ -168,7 +168,7 @@ public void ValidateReference() { return; } - if (Reference.IndexOf(":") != -1) + if (Reference.IndexOf(':') != -1) { ValidateReferenceAsDigest(); return; @@ -202,7 +202,13 @@ public string Digest() return Reference; } - public static void VerifyContentDigest(HttpResponseMessage resp, string refDigest) + /// + /// VerifyContentDigest verifies the content digest of the artifact. + /// + /// + /// + /// + internal static void VerifyContentDigest(HttpResponseMessage resp, string refDigest) { var digestStr = resp.Content.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); if (String.IsNullOrEmpty(digestStr)) diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs new file mode 100644 index 0000000..c46bf1c --- /dev/null +++ b/Oras/Remote/Registry.cs @@ -0,0 +1,64 @@ +using Oras.Exceptions; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using System.Threading; + +namespace Oras.Remote +{ + public class Registry + { + + public HttpClient HttpClient { get; set; } + public RemoteReference RemoteReference { get; set; } + public bool PlainHTTP { get; set; } + public string[] ManifestMediaTypes { get; set; } + public int TagListPageSize { get; set; } + public long MaxMetadataBytes { get; set; } + + /// + /// Client returns an HTTP client used to access the remote repository. + /// A default HTTP client is return if the client is not configured. + /// + /// + private HttpClient Client() + { + if (HttpClient is null) + { + var client = new HttpClient(); + client.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); + return client; + } + + return HttpClient; + } + + /// + /// PingAsync checks whether or not the registry implement Docker Registry API V2 or + /// OCI Distribution Specification. + /// Ping can be used to check authentication when an auth client is configured. + /// References: + /// - https://docs.docker.com/registry/spec/api/#base + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#api + /// + /// + /// + public async Task PingAsync(CancellationToken cancellationToken) + { + var url = URLUtiliity.BuildRegistryBaseURL(PlainHTTP, RemoteReference); + var resp = await Client().GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + return; + case HttpStatusCode.NotFound: + throw new NotFoundException($"Repository {RemoteReference} not found"); + default: + throw await ErrorUtil.ParseErrorResponse(resp); + } + } + } +} diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index bbb6db2..e7240fa 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -1,24 +1,25 @@ using Oras.Exceptions; using Oras.Interfaces.Registry; using Oras.Models; +using Oras.Utils; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using static System.Web.HttpUtility; using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using System.Web; - namespace Oras.Remote { public class RepositoryOption : IRepositoryOption { - public HttpClient Client { get; set; } - public ReferenceObj Reference { get; set; } + public HttpClient HttpClient { get; set; } + public RemoteReference RemoteReference { get; set; } public bool PlainHTTP { get; set; } public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } @@ -37,14 +38,14 @@ public class Repository : IRepository, IRepositoryOption public long defaultMaxMetaBytes = 4 * 1024 * 1024; //4 Mib /// - /// Client is the underlying HTTP client used to access the remote registry. + /// HttpClient is the underlying HTTP client used to access the remote registry. /// - public HttpClient Client { get; set; } + public HttpClient HttpClient { get; set; } /// /// ReferenceObj references the remote repository. /// - public ReferenceObj Reference { get; set; } + public RemoteReference RemoteReference { get; set; } /// /// PlainHTTP signals the transport to access the remote repository via HTTP @@ -91,8 +92,8 @@ public class Repository : IRepository, IRepositoryOption /// public Repository(string reference) { - var refObj = new ReferenceObj().ParseReference(reference); - Reference = refObj; + var refObj = new RemoteReference().ParseReference(reference); + RemoteReference = refObj; } /// @@ -102,13 +103,13 @@ public Repository(string reference) /// to multiple Repositories. To handle this we explicitly copy only the /// fields that we want to reproduce. /// - /// + /// /// - public Repository(ReferenceObj refObj, IRepositoryOption option) + public Repository(RemoteReference @ref, IRepositoryOption option) { - refObj.ValidateRepository(); - Client = option.Client; - Reference = refObj; + @ref.ValidateRepository(); + HttpClient = option.HttpClient; + RemoteReference = @ref; PlainHTTP = option.PlainHTTP; ManifestMediaTypes = option.ManifestMediaTypes; TagListPageSize = option.TagListPageSize; @@ -117,46 +118,23 @@ public Repository(ReferenceObj refObj, IRepositoryOption option) } /// - /// client returns an HTTP client used to access the remote repository. + /// Client returns an HTTP client used to access the remote repository. /// A default HTTP client is return if the client is not configured. /// /// - private HttpClient client() + private HttpClient Client() { - if (Client is null) + if (HttpClient is null) { var client = new HttpClient(); client.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); return client; } - return Client; + return HttpClient; } - /// - /// PingAsync checks whether or not the registry implement Docker Registry API V2 or - /// OCI Distribution Specification. - /// Ping can be used to check authentication when an auth client is configured. - /// References: - /// - https://docs.docker.com/registry/spec/api/#base - /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#api - /// - /// - /// - public async Task PingAsync(CancellationToken cancellationToken) - { - var url = RegistryUtil.BuildRegistryBaseURL(PlainHTTP, Reference); - var resp = await client().GetAsync(url, cancellationToken); - switch (resp.StatusCode) - { - case HttpStatusCode.OK: - return; - case HttpStatusCode.NotFound: - throw new NotFoundException($"Repository {Reference} not found"); - default: - throw await ErrorUtil.ParseErrorResponse(resp); - } - } + /// /// blobStore detects the blob store for the given descriptor. /// @@ -273,9 +251,9 @@ public async Task> TagsAsync(ITagLister repo, CancellationToken can var res = new List(); await repo.TagsAsync( string.Empty, - (tag) => + (tags) => { - res.AddRange(tag); + res.AddRange(tags); }, cancellationToken); return res; @@ -298,14 +276,14 @@ public async Task TagsAsync(string last, Action fn, CancellationToken { try { - var url = RegistryUtil.BuildRepositoryTagListURL(PlainHTTP, Reference); + var url = URLUtiliity.BuildRepositoryTagListURL(PlainHTTP, RemoteReference); while (true) { - url = await tagsAsync(last, fn, url, cancellationToken); - last = ""; + url = await TagsPageAsync(last, fn, url, cancellationToken); + last = ""; } } - catch (NoLinkHeaderException) + catch (Utils.NoLinkHeaderException) { return; } @@ -313,14 +291,14 @@ public async Task TagsAsync(string last, Action fn, CancellationToken } /// - /// tagsAsync returns a single page of tag list with the next link. + /// TagsPageAsync returns a single page of tag list with the next link. /// /// /// /// /// /// - private async Task tagsAsync(string last, Action fn, string url, CancellationToken cancellationToken) + private async Task TagsPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { if (PlainHTTP) { @@ -336,8 +314,8 @@ private async Task tagsAsync(string last, Action fn, string ur url = "https://" + url; } } - var uri = new UriBuilder(url); - var query = System.Web.HttpUtility.ParseQueryString(uri.Query); + var uriBuilder = new UriBuilder(url); + var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") { if (TagListPageSize > 0) @@ -352,21 +330,21 @@ private async Task tagsAsync(string last, Action fn, string ur } } - uri.Query = query.ToString(); - var resp = await Client.GetAsync(uri.ToString(), cancellationToken); + uriBuilder.Query = query.ToString(); + var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { throw await ErrorUtil.ParseErrorResponse(resp); } var data = await resp.Content.ReadAsStringAsync(); - var page = JsonSerializer.Deserialize(data); - fn(page.tags); + var tags = JsonSerializer.Deserialize (data); + fn(tags); return Utils.ParseLink(resp); } /// - /// deleteAsync removes the content identified by the descriptor in the + /// DeleteAsync removes the content identified by the descriptor in the /// entity blobs or manifests. /// /// @@ -375,23 +353,23 @@ private async Task tagsAsync(string last, Action fn, string ur /// /// /// - internal async Task deleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) + internal async Task DeleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) { - var refObj = Reference; + var refObj = RemoteReference; refObj.Reference = target.Digest; - Func buildURL = RegistryUtil.BuildRepositoryBlobURL; + Func buildURL = URLUtiliity.BuildRepositoryBlobURL; if (isManifest) { - buildURL = RegistryUtil.BuildRepositoryManifestURL; + buildURL = URLUtiliity.BuildRepositoryManifestURL; } var url = buildURL(PlainHTTP, refObj); - var resp = await Client.DeleteAsync(url, cancellationToken); + var resp = await HttpClient.DeleteAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.Accepted: - verifyContentDigest(resp, target.Digest); + VerifyContentDigest(resp, target.Digest); break; case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); @@ -402,14 +380,14 @@ internal async Task deleteAsync(Descriptor target, bool isManifest, Cancellation /// - /// verifyContentDigest verifies "Docker-Content-Digest" header if present. + /// VerifyContentDigest verifies "Docker-Content-Digest" header if present. /// OCI distribution-spec states the Docker-Content-Digest header is optional. /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#legacy-docker-support-http-headers /// /// /// /// - private void verifyContentDigest(HttpResponseMessage resp, string expected) + private void VerifyContentDigest(HttpResponseMessage resp, string expected) { resp.Content.Headers.TryGetValues(DockerContentDigestHeader, out var digestStr); if (digestStr == null || !digestStr.Any()) @@ -420,7 +398,7 @@ private void verifyContentDigest(HttpResponseMessage resp, string expected) string contentDigest; try { - contentDigest = DigestUtil.Parse(digestStr.FirstOrDefault()); + contentDigest = DigestUtility.Parse(digestStr.FirstOrDefault()); } catch (Exception) { @@ -470,18 +448,18 @@ public IManifestStore Manifests() /// /// /// - public ReferenceObj ParseReference(string reference) + public RemoteReference ParseReference(string reference) { try { - var refObj = new ReferenceObj().ParseReference(reference); - if (refObj.Registry != Reference.Registry || refObj.Repository != Reference.Repository) + var refObj = new RemoteReference().ParseReference(reference); + if (refObj.Registry != RemoteReference.Registry || refObj.Repository != RemoteReference.Repository) { throw new InvalidReferenceException( - $"mismatch between received {JsonSerializer.Serialize(refObj)} and expected {JsonSerializer.Serialize(Reference)}"); + $"mismatch between received {JsonSerializer.Serialize(refObj)} and expected {JsonSerializer.Serialize(RemoteReference)}"); } - if (refObj.Reference.Length == 0) + if (string.IsNullOrEmpty(refObj.Reference)) { throw new InvalidReferenceException(); } @@ -489,10 +467,10 @@ public ReferenceObj ParseReference(string reference) } catch (Exception) { - var refObj = new ReferenceObj + var refObj = new RemoteReference { - Registry = Reference.Registry, - Repository = Reference.Repository, + Registry = RemoteReference.Registry, + Repository = RemoteReference.Repository, Reference = reference }; //reference is not a FQDN @@ -532,7 +510,7 @@ public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); } - ReferenceObj.VerifyContentDigest(resp, refDigest); + RemoteReference.VerifyContentDigest(resp, refDigest); return new Descriptor { @@ -546,10 +524,10 @@ public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string public class ManifestStore : IManifestStore { - public Repository Repo { get; set; } + public Repository Repository { get; set; } public ManifestStore(Repository repository) { - Repo = repository; + Repository = repository; } @@ -563,12 +541,12 @@ public ManifestStore(Repository repository) /// public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var refObj = Repo.Reference; + var refObj = Repository.RemoteReference; refObj.Reference = target.Digest; - var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", target.MediaType); - var resp = await Repo.Client.SendAsync(req, cancellationToken); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { @@ -590,7 +568,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } - ReferenceObj.VerifyContentDigest(resp, target.Digest); + RemoteReference.VerifyContentDigest(resp, target.Digest); return await resp.Content.ReadAsStreamAsync(); } @@ -623,89 +601,42 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - await pushWithIndexing(expected, content, expected.Digest, cancellationToken); + await PushAsync(expected, content, expected.Digest, cancellationToken); } - /// - /// pushWithIndexing pushes the manifest content matching the expected descriptor. - /// - /// - /// - /// - /// - private async Task pushWithIndexing(Descriptor expected, Stream r, string reference, CancellationToken cancellationToken) - { - await pushAsync(expected, r, reference, cancellationToken); - return; - } /// - /// pushAsync pushes the manifest content, matching the expected descriptor. + /// PushAsync pushes the manifest content, matching the expected descriptor. /// /// /// /// /// - private async Task pushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) + private async Task PushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) { - var refObj = Repo.Reference; + var refObj = Repository.RemoteReference; refObj.Reference = reference; - // pushing usually requires both pull and push actions. - // Reference: https://github.com/distribution/distribution/blob/v2.7.1/registry/handlers/app.go#L921-L930 - var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Put, url); req.Content = new StreamContent(stream); - if (req.Content != null && req.Content.Headers.ContentLength != expected.Size) - { - // short circuit a size mismatch for built-in types - throw new Exception( - $"{req.Method} {req.RequestUri}: mismatch Content-Length: expect {expected.Size}"); - } req.Content.Headers.ContentLength = expected.Size; req.Content.Headers.Add("Content-Type", expected.MediaType); - - // if the underlying client is an auth client, the content might be read - // more than once for obtaining the auth challenge and the actual request. - // To prevent double reading, the manifest is read and stored in the memory, - // and serve from the memory. - var client = Repo.Client; + var client = Repository.HttpClient; var resp = await client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { throw await ErrorUtil.ParseErrorResponse(resp); } - ReferenceObj.VerifyContentDigest(resp, expected.Digest); - } - - /// - /// LimitSize returns ErrSizeExceedsLimit if the size of desc exceeds the limit n. - /// If n is less than or equal to zero, defaultMaxMetadataBytes is used. - /// - /// - /// - /// - private void LimitSize(Descriptor desc, long n) - { - if (n <= 0) - { - n = Repo.defaultMaxMetaBytes; - } - - if (desc.Size > n) - { - throw new SizeExceedsLimitException($"content size {desc.Size} exceeds MaxMetadataBytes {n}"); - } - - return; + RemoteReference.VerifyContentDigest(resp, expected.Digest); } public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); - var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var refObj = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Head, url); - req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); - var res = await Repo.Client.SendAsync(req, cancellationToken); + req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + var res = await Repository.HttpClient.SendAsync(req, cancellationToken); return res.StatusCode switch { @@ -716,14 +647,14 @@ public async Task ResolveAsync(string reference, CancellationToken c } /// - /// generateDescriptor returns a descriptor generated from the response. + /// GenerateDescriptor returns a descriptor generated from the response. /// /// - /// + /// /// /// /// - public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refObj, HttpMethod httpMethod) + public Descriptor GenerateDescriptor(HttpResponseMessage res, RemoteReference @ref, HttpMethod httpMethod) { string mediaType; try @@ -731,7 +662,6 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refOb // 1. Validate Content-Type mediaType = res.Content.Headers.ContentType.MediaType; MediaTypeHeaderValue.Parse(mediaType); - } catch (Exception e) { @@ -748,19 +678,20 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refOb string refDigest = string.Empty; try { - refDigest = refObj.Digest(); + refDigest = @ref.Digest(); } - catch (Exception e) + catch (Exception) { } // 4. Validate Server Digest (if present) res.Content.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); - if (!string.IsNullOrEmpty(serverHeaderDigest.First())) + var serverDigest = serverHeaderDigest.First(); + if (!string.IsNullOrEmpty(serverDigest)) { try { - ReferenceObj.VerifyContentDigest(res, serverHeaderDigest.FirstOrDefault()); + RemoteReference.VerifyContentDigest(res, serverDigest); } catch (Exception) { @@ -771,11 +702,11 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refOb // 5. Now, look for specific error conditions; string contentDigest; - if (serverHeaderDigest.First().Length == 0) + if (string.IsNullOrEmpty(serverDigest)) { if (httpMethod == HttpMethod.Head) { - if (refDigest.Length == 0) + if (string.IsNullOrEmpty(refDigest)) { // HEAD without server `Docker-Content-Digest` // immediate fail @@ -791,7 +722,7 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refOb string calculatedDigest; try { - calculatedDigest = calculateDigestFromResponse(res, Repo.MaxMetadataBytes); + calculatedDigest = calculateDigestFromResponse(res, Repository.MaxMetadataBytes); } catch (Exception e) { @@ -802,7 +733,7 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, ReferenceObj refOb } else { - contentDigest = serverHeaderDigest.FirstOrDefault(); + contentDigest = serverDigest; } if (refDigest.Length > 0 && refDigest != contentDigest) { @@ -835,7 +766,7 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta { throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {ex.Message}"); } - return DigestUtil.FromBytes(content); + return DigestUtility.FromBytes(content); } /// @@ -846,19 +777,9 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - await deleteWithIndexingAsync(target, cancellationToken); - } - - /// - /// deleteWithIndexingAsync removes the manifest content identified by the descriptor. - /// - /// - /// - /// - private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken cancellationToken) - { - await Repo.deleteAsync(target, true, cancellationToken); + await Repository.DeleteAsync(target, true, cancellationToken); } + /// /// FetchReferenceAsync fetches the manifest identified by the reference. @@ -868,11 +789,11 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken /// public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); - var url = RegistryUtil.BuildRepositoryManifestURL(Repo.PlainHTTP, refObj); + var refObj = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repo.ManifestMediaTypes)); - var resp = await Repo.Client.SendAsync(req, cancellationToken); + req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -905,8 +826,8 @@ private async Task deleteWithIndexingAsync(Descriptor target, CancellationToken public async Task PushReferenceAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); - await pushWithIndexing(expected, content, refObj.Reference, cancellationToken); + var refObj = Repository.ParseReference(reference); + await PushAsync(expected, content, refObj.Reference, cancellationToken); } /// @@ -918,30 +839,31 @@ public async Task PushReferenceAsync(Descriptor expected, Stream content, string /// public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); + var refObj = Repository.ParseReference(reference); var rc = await FetchAsync(descriptor, cancellationToken); - await pushAsync(descriptor, rc, refObj.Reference, cancellationToken); + await PushAsync(descriptor, rc, refObj.Reference, cancellationToken); } } - public class BlobStore : IBlobStore + internal class BlobStore : IBlobStore { - public Repository Repo { get; set; } + public Repository Repository { get; set; } public BlobStore(Repository repository) { - Repo = repository; + Repository = repository; } public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var refObj = Repo.Reference; + var refObj = Repository.RemoteReference; + DigestUtility.Parse(target.Digest); refObj.Reference = target.Digest; - var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); - var resp = await Repo.Client.GetAsync(url, cancellationToken); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -968,8 +890,8 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel { to = target.Size; } - Repo.Client.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); - resp = await Repo.Client.GetAsync(url, cancellationToken); + Repository.HttpClient.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); + resp = await Repository.HttpClient.GetAsync(url, cancellationToken); if (resp.StatusCode != HttpStatusCode.PartialContent) { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); @@ -1027,8 +949,8 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - var url = RegistryUtil.BuildRepositoryBlobUploadURL(Repo.PlainHTTP, Repo.Reference); - var resp = await Repo.Client.PostAsync(url, null, cancellationToken); + var url = URLUtiliity.BuildRepositoryBlobUploadURL(Repository.PlainHTTP, Repository.RemoteReference); + var resp = await Repository.HttpClient.PostAsync(url, null, cancellationToken); var reqHostname = resp.RequestMessage.RequestUri.Host; var reqPort = resp.RequestMessage.RequestUri.Port; if (resp.StatusCode != HttpStatusCode.Accepted) @@ -1074,7 +996,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok req.Content.Headers.Add("Content-Type", "application/octet-stream"); // add digest key to query string with expected digest value - var query = HttpUtility.ParseQueryString(new Uri(location).Query); + var query = ParseQueryString(new Uri(location).Query); query.Add("digest", expected.Digest); req.RequestUri = new Uri($"{req.RequestUri}?digest={expected.Digest}"); @@ -1084,7 +1006,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok { req.Headers.Add("Authorization", auth.FirstOrDefault()); } - resp = await Repo.Client.SendAsync(req, cancellationToken); + resp = await Repository.HttpClient.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { throw await ErrorUtil.ParseErrorResponse(resp); @@ -1101,11 +1023,11 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok /// public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); + var refObj = Repository.ParseReference(reference); var refDigest = refObj.Digest(); - var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); - var resp = await Repo.Client.SendAsync(requestMessage, cancellationToken); + var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); return resp.StatusCode switch { HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), @@ -1122,7 +1044,7 @@ public async Task ResolveAsync(string reference, CancellationToken c /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - await Repo.deleteAsync(target, false, cancellationToken); + await Repository.DeleteAsync(target, false, cancellationToken); } /// @@ -1134,10 +1056,10 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT /// public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repo.ParseReference(reference); + var refObj = Repository.ParseReference(reference); var refDigest = refObj.Digest(); - var url = RegistryUtil.BuildRepositoryBlobURL(Repo.PlainHTTP, refObj); - var resp = await Repo.Client.GetAsync(url, cancellationToken); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -1168,8 +1090,8 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { to = desc.Size; } - Repo.Client.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); - resp = await Repo.Client.GetAsync(url, cancellationToken); + Repository.HttpClient.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); + resp = await Repository.HttpClient.GetAsync(url, cancellationToken); if (resp.StatusCode != HttpStatusCode.PartialContent) { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs deleted file mode 100644 index e0b29c7..0000000 --- a/Oras/Remote/ResponseTypes.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Oras.Remote -{ - internal static class ResponseTypes - { - public class Tags - { - public string[] tags { get; set; } - } - } -} diff --git a/Oras/Remote/RegistryUtil.cs b/Oras/Remote/URLUtiliity.cs similarity index 64% rename from Oras/Remote/RegistryUtil.cs rename to Oras/Remote/URLUtiliity.cs index fbe222e..cc3b26b 100644 --- a/Oras/Remote/RegistryUtil.cs +++ b/Oras/Remote/URLUtiliity.cs @@ -1,14 +1,14 @@ namespace Oras.Remote { - internal static class RegistryUtil + internal static class URLUtiliity { /// /// BuildScheme returns HTTP scheme used to access the remote registry. /// /// /// - public static string BuildScheme(bool plainHTTP) + internal static string BuildScheme(bool plainHTTP) { if (plainHTTP) { @@ -24,11 +24,11 @@ public static string BuildScheme(bool plainHTTP) /// Reference: https://docs.docker.com/registry/spec/api/#base /// /// - /// + /// /// - public static string BuildRegistryBaseURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRegistryBaseURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/"; + return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/"; } /// @@ -37,11 +37,11 @@ public static string BuildRegistryBaseURL(bool plainHTTP, ReferenceObj refObj) /// Reference: https://docs.docker.com/registry/spec/api/#catalog /// /// - /// + /// /// - public static string BuildRegistryCatalogURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRegistryCatalogURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/_catalog"; + return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/_catalog"; } /// @@ -49,11 +49,11 @@ public static string BuildRegistryCatalogURL(bool plainHTTP, ReferenceObj refObj /// Format: :///v2/ /// /// - /// + /// /// - public static string BuildRepositoryBaseURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRepositoryBaseURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}"; + return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/{@ref.Repository}"; } /// @@ -62,11 +62,11 @@ public static string BuildRepositoryBaseURL(bool plainHTTP, ReferenceObj refObj) /// Reference: https://docs.docker.com/registry/spec/api/#tags /// /// - /// + /// /// - public static string BuildRepositoryTagListURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRepositoryTagListURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildScheme(plainHTTP)}://{refObj.Host()}/v2/{refObj.Repository}/tags/list"; + return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/{@ref.Repository}/tags/list"; } /// @@ -75,11 +75,11 @@ public static string BuildRepositoryTagListURL(bool plainHTTP, ReferenceObj refO /// Reference: https://docs.docker.com/registry/spec/api/#manifest /// /// - /// + /// /// - public static string BuildRepositoryManifestURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRepositoryManifestURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/manifests/{refObj.Reference}"; + return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/manifests/{@ref.Reference}"; } /// @@ -88,11 +88,11 @@ public static string BuildRepositoryManifestURL(bool plainHTTP, ReferenceObj ref /// Reference: https://docs.docker.com/registry/spec/api/#blob /// /// - /// + /// /// - public static string BuildRepositoryBlobURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRepositoryBlobURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/blobs/{refObj.Reference}"; + return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/blobs/{@ref.Reference}"; } /// @@ -102,11 +102,11 @@ public static string BuildRepositoryBlobURL(bool plainHTTP, ReferenceObj refObj) /// /// - /// + /// /// - public static string BuildRepositoryBlobUploadURL(bool plainHTTP, ReferenceObj refObj) + internal static string BuildRepositoryBlobUploadURL(bool plainHTTP, RemoteReference @ref) { - return $"{BuildRepositoryBaseURL(plainHTTP, refObj)}/blobs/uploads/"; + return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/blobs/uploads/"; } } diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index 771c04b..41d9906 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -59,7 +59,7 @@ public static string ParseLink(HttpResponseMessage resp) } /// - /// LimitReader returns a Reader that reads from content but stops after n + /// LimitReader ensures that the read byte does not exceed n /// bytes. if n is less than or equal to zero, defaultMaxMetadataBytes is used. /// /// @@ -82,5 +82,25 @@ public static byte[] LimitReader(HttpContent content, long n) return bytes; } + + /// + /// NoLinkHeaderException is thrown when a link header is missing. + /// + public class NoLinkHeaderException : Exception + { + public NoLinkHeaderException() + { + } + + public NoLinkHeaderException(string message) + : base(message) + { + } + + public NoLinkHeaderException(string message, Exception inner) + : base(message, inner) + { + } + } } } diff --git a/Oras/Remote/DigestUtil.cs b/Oras/Utils/DigestUtility.cs similarity index 87% rename from Oras/Remote/DigestUtil.cs rename to Oras/Utils/DigestUtility.cs index 8afc421..2b89e01 100644 --- a/Oras/Remote/DigestUtil.cs +++ b/Oras/Utils/DigestUtility.cs @@ -1,12 +1,13 @@ using Oras.Exceptions; +using Oras.Remote; using System; using System.Net.Http; using System.Security.Cryptography; using System.Text.RegularExpressions; -namespace Oras.Remote +namespace Oras.Utils { - internal class DigestUtil + internal class DigestUtility { /// /// Parse verifies the digest header and throws an exception if it is invalid. @@ -14,7 +15,7 @@ internal class DigestUtil /// public static string Parse(string digest) { - if (!Regex.IsMatch(digest, ReferenceObj.digestRegexp)) + if (!Regex.IsMatch(digest, RemoteReference.digestRegexp)) { throw new InvalidReferenceException($"invalid reference format: {digest}"); } From 6c74d76b07e40ffde140f9c89631495424f57cdd Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 04:30:55 +0100 Subject: [PATCH 48/77] formatted code Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 49 ++++++++-------- Oras/Remote/ErrorUtil.cs | 2 +- Oras/Remote/Registry.cs | 9 +-- Oras/Remote/Repository.cs | 91 +++++------------------------ Oras/Remote/Utils.cs | 7 +-- Oras/Utils/DigestUtility.cs | 1 - 6 files changed, 45 insertions(+), 114 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 61bfec7..444c980 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -1,22 +1,19 @@ -using System.Collections.Immutable; -using Moq; +using Moq; using Moq.Protected; using Oras.Constants; using Oras.Exceptions; using Oras.Models; using Oras.Remote; +using System.Collections.Immutable; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; -using Oras.Remote; using System.Text.RegularExpressions; +using System.Web; using Xunit; using static Oras.Content.Content; -using System.Web; -using Xunit.Abstractions; -using System; namespace Oras.Tests.RemoteTest { @@ -31,7 +28,7 @@ public struct TestIOStruct public bool errExpectedOnGET; } - private byte[] theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); + private byte[] theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); private const string theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string dockerContentDigestHeader = "Docker-Content-Digest"; @@ -135,7 +132,7 @@ public async Task Repository_FetchAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() @@ -210,7 +207,7 @@ public async Task Repository_PushAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() @@ -302,7 +299,7 @@ public async Task Repository_ExistsAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() @@ -366,7 +363,7 @@ public async Task Repository_DeleteAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var blobDeleted = false; var index = @"{""manifests"":[]}"u8.ToArray(); @@ -426,7 +423,7 @@ public async Task Repository_ResolveAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() @@ -500,7 +497,7 @@ public async Task Repository_TagAsync() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() @@ -625,7 +622,7 @@ public async Task Repository_FetchReferenceAsyc() { Digest = CalculateDigest(blob), MediaType = "test", - Size = (uint) blob.Length + Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() @@ -907,7 +904,7 @@ public async Task BlobStore_FetchAsync_CanSeek() } res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int) start..(int) end]); + res.Content = new ByteArrayContent(blob[(int)start..(int)end]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return res; @@ -1347,7 +1344,7 @@ public async Task BlobStore_FetchReferenceAsync_Seek() } res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int) start..]); + res.Content = new ByteArrayContent(blob[(int)start..]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Docker-Content-Digest", blobDesc.Digest); return res; @@ -1430,7 +1427,7 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { Method = method }; - + } else { @@ -1443,7 +1440,7 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() var d = string.Empty; try { - d = reference.Digest(); + d = reference.Digest(); } catch (Exception e) { @@ -1463,7 +1460,7 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() try { Repository.GenerateBlobDescriptor(resp, d); - + } catch (Exception e) { @@ -1473,7 +1470,7 @@ public async Task GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() throw new Exception( $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } - + } if (errExpected && !err) @@ -1510,7 +1507,7 @@ public async Task ManifestStore_FetchAsync() } if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { - if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) + if (req.Headers.TryGetValues("Accept", out IEnumerable values) && !values.Contains(OCIMediaTypes.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } @@ -1530,7 +1527,7 @@ public async Task ManifestStore_FetchAsync() var buf = new byte[data.Length]; await data.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); - + var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { @@ -1915,7 +1912,7 @@ public async Task ManifestStore_TagAsync() repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - + Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); await store.TagAsync(indexDesc, reference, cancellationToken); @@ -1974,7 +1971,7 @@ public async Task ManifestStore_PushReferenceAsync() Assert.Equal(index, gotIndex); } - + public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { var reference = new RemoteReference() @@ -1986,7 +1983,7 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { - var repo = new Repository(reference.Repository+"/"+reference.Repository); + var repo = new Repository(reference.Repository + "/" + reference.Repository); HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; var s = new ManifestStore(repo); foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) @@ -2014,7 +2011,7 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest var err = false; try { - s.GenerateDescriptor(resp, reference,method); + s.GenerateDescriptor(resp, reference, method); } catch (Exception e) { diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtil.cs index c94d853..04642aa 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtil.cs @@ -14,7 +14,7 @@ internal class ErrorUtil internal static async Task ParseErrorResponse(HttpResponseMessage response) { var body = await response.Content.ReadAsStringAsync(); - return new Exception( new + return new Exception(new { response.RequestMessage.Method, URL = response.RequestMessage.RequestUri, diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index c46bf1c..74b8eda 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -1,11 +1,8 @@ using Oras.Exceptions; -using System; -using System.Collections.Generic; -using System.Net.Http; using System.Net; -using System.Text; -using System.Threading.Tasks; +using System.Net.Http; using System.Threading; +using System.Threading.Tasks; namespace Oras.Remote { @@ -18,7 +15,7 @@ public class Registry public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } public long MaxMetadataBytes { get; set; } - + /// /// Client returns an HTTP client used to access the remote repository. /// A default HTTP client is return if the client is not configured. diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index e7240fa..e20c6c8 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -7,13 +7,12 @@ using System.IO; using System.Linq; using System.Net; -using static System.Web.HttpUtility; using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using System.Web; +using static System.Web.HttpUtility; namespace Oras.Remote { public class RepositoryOption : IRepositoryOption @@ -134,7 +133,7 @@ private HttpClient Client() return HttpClient; } - + /// /// blobStore detects the blob store for the given descriptor. /// @@ -279,8 +278,8 @@ public async Task TagsAsync(string last, Action fn, CancellationToken var url = URLUtiliity.BuildRepositoryTagListURL(PlainHTTP, RemoteReference); while (true) { - url = await TagsPageAsync(last, fn, url, cancellationToken); - last = ""; + url = await TagsPageAsync(last, fn, url, cancellationToken); + last = ""; } } catch (Utils.NoLinkHeaderException) @@ -321,8 +320,8 @@ private async Task TagsPageAsync(string last, Action fn, strin if (TagListPageSize > 0) { query["n"] = TagListPageSize.ToString(); - - + + } if (last != "") { @@ -338,7 +337,7 @@ private async Task TagsPageAsync(string last, Action fn, strin } var data = await resp.Content.ReadAsStringAsync(); - var tags = JsonSerializer.Deserialize (data); + var tags = JsonSerializer.Deserialize(data); fn(tags); return Utils.ParseLink(resp); } @@ -760,7 +759,7 @@ private string calculateDigestFromResponse(HttpResponseMessage res, long maxMeta byte[] content = null; try { - content = Utils.LimitReader(res.Content, maxMetadataBytes); + content = Utils.LimitReader(res.Content, maxMetadataBytes); } catch (Exception ex) { @@ -779,7 +778,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { await Repository.DeleteAsync(target, true, cancellationToken); } - + /// /// FetchReferenceAsync fetches the manifest identified by the reference. @@ -873,36 +872,6 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } - // check server range request capability. - // Docker spec allows range header form of "Range: bytes=-". - // However, the remote server may still not RFC 7233 compliant. - // Reference: https://docs.docker.com/registry/spec/api/#blob - if (resp.Headers.TryGetValues("Accept-Ranges", out var acceptRanges) && acceptRanges.FirstOrDefault() == "bytes") - { - var stream = new MemoryStream(); - long from = 0; - // make request using ranges until the whole data is read - while (from < target.Size) - { - - var to = from + 1024 * 1024 - 1; - if (to > target.Size) - { - to = target.Size; - } - Repository.HttpClient.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); - resp = await Repository.HttpClient.GetAsync(url, cancellationToken); - if (resp.StatusCode != HttpStatusCode.PartialContent) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); - } - await resp.Content.CopyToAsync(stream); - from = to + 1; - } - stream.Seek(0, SeekOrigin.Begin); - return stream; - } - return await resp.Content.ReadAsStreamAsync(); case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); @@ -924,7 +893,7 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell await ResolveAsync(target.Digest, cancellationToken); return true; } - catch (Exception ex) when (ex is NotFoundException) + catch (NotFoundException) { return false; } @@ -958,11 +927,11 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok throw await ErrorUtil.ParseErrorResponse(resp); } - string location = String.Empty; + string location; // monolithic upload if (!resp.Headers.Location.IsAbsoluteUri) { - location = resp.RequestMessage.RequestUri.Scheme+"://"+ resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; + location = resp.RequestMessage.RequestUri.Scheme + "://" + resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; } else { @@ -974,14 +943,14 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok // the hostname of the Location header in the response is set to // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. var uri = new Uri(location); - var locationHostname = uri.Host ; + var locationHostname = uri.Host; var locationPort = uri.Port; // if location port 443 is missing, add it back if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) { location = new Uri($"{locationHostname}:{reqPort}").ToString(); } - + url = location; var req = new HttpRequestMessage(HttpMethod.Put, url); @@ -1073,45 +1042,15 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = Repository.GenerateBlobDescriptor(resp, refDigest); } - // check server range request capability. - // Docker spec allows range header form of "Range: bytes=-". - // However, the remote server may still not RFC 7233 compliant. - // Reference: https://docs.docker.com/registry/spec/api/#blob - if (resp.Headers.TryGetValues("Accept-Ranges", out var acceptRanges) && acceptRanges.FirstOrDefault() == "bytes") - { - var stream = new MemoryStream(); - // make request using ranges until the whole data is read - long from = 0; - - while (from < desc.Size) - { - var to = from + 1024 * 1024 - 1; - if (to > desc.Size) - { - to = desc.Size; - } - Repository.HttpClient.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to); - resp = await Repository.HttpClient.GetAsync(url, cancellationToken); - if (resp.StatusCode != HttpStatusCode.PartialContent) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response status code: {resp.StatusCode}"); - } - await resp.Content.CopyToAsync(stream); - from = to + 1; - } - stream.Seek(0, SeekOrigin.Begin); - return (desc, stream); - } + return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException(); default: throw await ErrorUtil.ParseErrorResponse(resp); - } } - } } diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index 41d9906..f1337fd 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -1,5 +1,4 @@ -using Oras.Exceptions; -using System; +using System; using System.Linq; using System.Net.Http; @@ -26,7 +25,7 @@ public static string ParseLink(HttpResponseMessage resp) { link = values.FirstOrDefault(); } - else + else { throw new NoLinkHeaderException(); } @@ -52,7 +51,7 @@ public static string ParseLink(HttpResponseMessage resp) var scheme = resp.RequestMessage.RequestUri.Scheme; var authority = resp.RequestMessage.RequestUri.Authority; - Uri baseUri = new Uri(scheme+"://"+authority); + Uri baseUri = new Uri(scheme + "://" + authority); Uri resolvedUri = new Uri(baseUri, link); return resolvedUri.AbsoluteUri; diff --git a/Oras/Utils/DigestUtility.cs b/Oras/Utils/DigestUtility.cs index 2b89e01..b5ad574 100644 --- a/Oras/Utils/DigestUtility.cs +++ b/Oras/Utils/DigestUtility.cs @@ -1,7 +1,6 @@ using Oras.Exceptions; using Oras.Remote; using System; -using System.Net.Http; using System.Security.Cryptography; using System.Text.RegularExpressions; From ab4e8fa9122ce536946aaa75aaf8f42ff5b812b2 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 04:32:30 +0100 Subject: [PATCH 49/77] fix: changed string to character Signed-off-by: Samson Amaugo --- Oras/Remote/ManifestUtil.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtil.cs index 7b51899..4d6a0e7 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtil.cs @@ -47,7 +47,7 @@ public static string ManifestAcceptHeader(string[] manifestMediaTypes) manifestMediaTypes = DefaultManifestMediaTypes; } - return string.Join(",", manifestMediaTypes); + return string.Join(',', manifestMediaTypes); } } } From cc6c64d93f1bba26bd2ffc6af95824dc47a7bbb6 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 08:33:55 +0100 Subject: [PATCH 50/77] made some changes Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RemoteTest.cs | 52 +++++++++++++++++------------ Oras/Models/Descriptor.cs | 15 +-------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 444c980..5b660d1 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -14,6 +14,7 @@ using System.Web; using Xunit; using static Oras.Content.Content; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace Oras.Tests.RemoteTest { @@ -109,6 +110,18 @@ public static Dictionary GetTestIOStructMapForGetDescripto } }; } + + /// + /// AreDescriptorsEqual compares two descriptors and returns true if they are equal. + /// + /// + /// + /// + public bool AreDescriptorsEqual(Descriptor a, Descriptor b) + { + return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; + } + public static HttpClient CustomClient(Func func) { var moqHandler = new Mock(); @@ -474,15 +487,16 @@ await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); - Assert.Equal(indexDesc, got); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + got = await repo.ResolveAsync(reference, cancellationToken); - Assert.Equal(indexDesc, got); + Assert.True(AreDescriptorsEqual(indexDesc, got)); var tagDigestRef = "whatever" + "@" + indexDesc.Digest; got = await repo.ResolveAsync(tagDigestRef, cancellationToken); - Assert.Equal(indexDesc, got); + Assert.True(AreDescriptorsEqual(indexDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await repo.ResolveAsync(fqdnRef, cancellationToken); - Assert.Equal(indexDesc, got); + Assert.True(AreDescriptorsEqual(indexDesc, got)); } /// @@ -675,16 +689,14 @@ await Assert.ThrowsAsync( // test with manifest digest var data = await repo.FetchReferenceAsync(indexDesc.Digest, cancellationToken); - Assert.Equal(indexDesc, data.Descriptor); - + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag data = await repo.FetchReferenceAsync(reference, cancellationToken); - Assert.Equal(indexDesc, data.Descriptor); - + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); @@ -692,8 +704,7 @@ await Assert.ThrowsAsync( // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; data = await repo.FetchReferenceAsync(tagDigestRef, cancellationToken); - Assert.Equal(indexDesc, data.Descriptor); - + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); @@ -701,7 +712,7 @@ await Assert.ThrowsAsync( // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await repo.FetchReferenceAsync(fqdnRef, cancellationToken); - Assert.Equal(indexDesc, data.Descriptor); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -1742,17 +1753,17 @@ public async Task ManifestStore_ResolveAsync() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); - Assert.Equal(manifestDesc, got); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); got = await store.ResolveAsync(reference, cancellationToken); - Assert.Equal(manifestDesc, got); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; got = await store.ResolveAsync(tagDigestRef, cancellationToken); - Assert.Equal(manifestDesc, got); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await store.ResolveAsync(fqdnRef, cancellationToken); - Assert.Equal(manifestDesc, got); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor @@ -1809,8 +1820,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with tag var data = await store.FetchReferenceAsync(reference, cancellationToken); - Assert.Equal(manifestDesc, data.Descriptor); - + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); @@ -1821,7 +1831,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with digest data = await store.FetchReferenceAsync(manifestDesc.Digest, cancellationToken); - Assert.Equal(manifestDesc, data.Descriptor); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -1830,7 +1840,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; data = await store.FetchReferenceAsync(tagDigestRef, cancellationToken); - Assert.Equal(manifestDesc, data.Descriptor); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); @@ -1838,7 +1848,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await store.FetchReferenceAsync(fqdnRef, cancellationToken); - Assert.Equal(manifestDesc, data.Descriptor); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); @@ -2032,4 +2042,4 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest } } -} \ No newline at end of file +} diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index 4eb899d..693a47b 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -5,7 +5,7 @@ namespace Oras.Models { - public class Descriptor : IEquatable + public class Descriptor { [JsonPropertyName("mediaType")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] @@ -46,19 +46,6 @@ internal static MinimumDescriptor FromOCI(Descriptor descriptor) Size = descriptor.Size }; } - - public bool Equals(Descriptor other) - { - if (other == null) return false; - return this.MediaType == other.MediaType && this.Digest == other.Digest && this.Size == other.Size; - } - - - - public override int GetHashCode() - { - return HashCode.Combine(MediaType, Digest, Size, URLs, Annotations, Data, Platform, ArtifactType); - } } public class Platform From 374ed5980e80ed49ae0c01f2a99454bb7a8c55e2 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 08:35:35 +0100 Subject: [PATCH 51/77] fix: changed members name Signed-off-by: Samson Amaugo --- Oras/Memory/MemoryGraph.cs | 6 +++--- Oras/Memory/MemoryStorage.cs | 6 +++--- Oras/Models/Descriptor.cs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Oras/Memory/MemoryGraph.cs b/Oras/Memory/MemoryGraph.cs index 99f5b0e..8e3859f 100644 --- a/Oras/Memory/MemoryGraph.cs +++ b/Oras/Memory/MemoryGraph.cs @@ -29,7 +29,7 @@ internal async Task IndexAsync(IFetcher fetcher, Descriptor node, CancellationTo /// internal async Task> PredecessorsAsync(Descriptor node, CancellationToken cancellationToken) { - var key = Descriptor.FromOCI(node); + var key = Descriptor.GetMinimum(node); if (!this._predecessors.TryGetValue(key, out ConcurrentDictionary predecessors)) { return default; @@ -52,10 +52,10 @@ private void Index(Descriptor node, IList successors, CancellationTo { return; } - var predecessorKey = Descriptor.FromOCI(node); + var predecessorKey = Descriptor.GetMinimum(node); foreach (var successor in successors) { - var successorKey = Descriptor.FromOCI(successor); + var successorKey = Descriptor.GetMinimum(successor); var predecessors = this._predecessors.GetOrAdd(successorKey, new ConcurrentDictionary()); predecessors.TryAdd(predecessorKey, node); } diff --git a/Oras/Memory/MemoryStorage.cs b/Oras/Memory/MemoryStorage.cs index b6a5332..aad0204 100644 --- a/Oras/Memory/MemoryStorage.cs +++ b/Oras/Memory/MemoryStorage.cs @@ -15,7 +15,7 @@ internal class MemoryStorage : IStorage public Task ExistsAsync(Descriptor target, CancellationToken cancellationToken) { - var contentExist = _content.ContainsKey(Descriptor.FromOCI(target)); + var contentExist = _content.ContainsKey(Descriptor.GetMinimum(target)); return Task.FromResult(contentExist); } @@ -23,7 +23,7 @@ public Task ExistsAsync(Descriptor target, CancellationToken cancellationT public Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var contentExist = this._content.TryGetValue(Descriptor.FromOCI(target), out byte[] content); + var contentExist = this._content.TryGetValue(Descriptor.GetMinimum(target), out byte[] content); if (!contentExist) { throw new NotFoundException($"{target.Digest} : {target.MediaType}"); @@ -34,7 +34,7 @@ public Task FetchAsync(Descriptor target, CancellationToken cancellation public async Task PushAsync(Descriptor expected, Stream contentStream, CancellationToken cancellationToken = default) { - var key = Descriptor.FromOCI(expected); + var key = Descriptor.GetMinimum(expected); var contentExist = _content.TryGetValue(key, out byte[] _); if (contentExist) { diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index 693a47b..82aff54 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -37,7 +37,7 @@ public class Descriptor [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string ArtifactType { get; set; } - internal static MinimumDescriptor FromOCI(Descriptor descriptor) + internal static MinimumDescriptor GetMinimum(Descriptor descriptor) { return new MinimumDescriptor { From bc76e685787134aa14f677efeb50f81eb84373f8 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 09:10:20 +0100 Subject: [PATCH 52/77] fix: changed members accessor Signed-off-by: Samson Amaugo --- Oras/Memory/MemoryGraph.cs | 7 ++++--- Oras/Memory/MemoryStorage.cs | 6 +++--- Oras/Models/Descriptor.cs | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Oras/Memory/MemoryGraph.cs b/Oras/Memory/MemoryGraph.cs index 8e3859f..c9cd9af 100644 --- a/Oras/Memory/MemoryGraph.cs +++ b/Oras/Memory/MemoryGraph.cs @@ -29,7 +29,7 @@ internal async Task IndexAsync(IFetcher fetcher, Descriptor node, CancellationTo /// internal async Task> PredecessorsAsync(Descriptor node, CancellationToken cancellationToken) { - var key = Descriptor.GetMinimum(node); + var key = node.GetMinimum(); if (!this._predecessors.TryGetValue(key, out ConcurrentDictionary predecessors)) { return default; @@ -52,10 +52,11 @@ private void Index(Descriptor node, IList successors, CancellationTo { return; } - var predecessorKey = Descriptor.GetMinimum(node); + + var predecessorKey = node.GetMinimum(); foreach (var successor in successors) { - var successorKey = Descriptor.GetMinimum(successor); + var successorKey = successor.GetMinimum(); var predecessors = this._predecessors.GetOrAdd(successorKey, new ConcurrentDictionary()); predecessors.TryAdd(predecessorKey, node); } diff --git a/Oras/Memory/MemoryStorage.cs b/Oras/Memory/MemoryStorage.cs index aad0204..b207051 100644 --- a/Oras/Memory/MemoryStorage.cs +++ b/Oras/Memory/MemoryStorage.cs @@ -15,7 +15,7 @@ internal class MemoryStorage : IStorage public Task ExistsAsync(Descriptor target, CancellationToken cancellationToken) { - var contentExist = _content.ContainsKey(Descriptor.GetMinimum(target)); + var contentExist = _content.ContainsKey(target.GetMinimum()); return Task.FromResult(contentExist); } @@ -23,7 +23,7 @@ public Task ExistsAsync(Descriptor target, CancellationToken cancellationT public Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var contentExist = this._content.TryGetValue(Descriptor.GetMinimum(target), out byte[] content); + var contentExist = this._content.TryGetValue(target.GetMinimum(), out byte[] content); if (!contentExist) { throw new NotFoundException($"{target.Digest} : {target.MediaType}"); @@ -34,7 +34,7 @@ public Task FetchAsync(Descriptor target, CancellationToken cancellation public async Task PushAsync(Descriptor expected, Stream contentStream, CancellationToken cancellationToken = default) { - var key = Descriptor.GetMinimum(expected); + var key = expected.GetMinimum(); var contentExist = _content.TryGetValue(key, out byte[] _); if (contentExist) { diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index 82aff54..82c7ede 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -37,13 +37,13 @@ public class Descriptor [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string ArtifactType { get; set; } - internal static MinimumDescriptor GetMinimum(Descriptor descriptor) + internal MinimumDescriptor GetMinimum() { return new MinimumDescriptor { - MediaType = descriptor.MediaType, - Digest = descriptor.Digest, - Size = descriptor.Size + MediaType = this.MediaType, + Digest = this.Digest, + Size = this.Size }; } } From 778fd0aaa42bf0490f50152029295860c20cae44 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 16:28:08 +0100 Subject: [PATCH 53/77] fix: changed members accessor Signed-off-by: Samson Amaugo --- Oras/{Utils => Content}/DigestUtility.cs | 14 ++- Oras/Interfaces/Registry/IRepositoryOption.cs | 10 +-- Oras/Remote/{ErrorUtil.cs => ErrorUtility.cs} | 2 +- .../{ManifestUtil.cs => ManifestUtility.cs} | 6 +- Oras/Remote/Reference.cs | 35 ++++---- Oras/Remote/Registry.cs | 89 ++++++++++++++++++- Oras/Remote/Repository.cs | 66 +++++--------- Oras/Remote/URLUtiliity.cs | 42 ++++----- Oras/Remote/Utils.cs | 40 +-------- 9 files changed, 168 insertions(+), 136 deletions(-) rename Oras/{Utils => Content}/DigestUtility.cs (68%) rename Oras/Remote/{ErrorUtil.cs => ErrorUtility.cs} (95%) rename Oras/Remote/{ManifestUtil.cs => ManifestUtility.cs} (87%) diff --git a/Oras/Utils/DigestUtility.cs b/Oras/Content/DigestUtility.cs similarity index 68% rename from Oras/Utils/DigestUtility.cs rename to Oras/Content/DigestUtility.cs index b5ad574..7113a70 100644 --- a/Oras/Utils/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -4,17 +4,23 @@ using System.Security.Cryptography; using System.Text.RegularExpressions; -namespace Oras.Utils +namespace Oras.Content { internal class DigestUtility { + + /// + /// digestRegexp checks the digest. + /// + public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; + /// /// Parse verifies the digest header and throws an exception if it is invalid. /// /// public static string Parse(string digest) { - if (!Regex.IsMatch(digest, RemoteReference.digestRegexp)) + if (!Regex.IsMatch(digest, digestRegexp)) { throw new InvalidReferenceException($"invalid reference format: {digest}"); } @@ -23,11 +29,11 @@ public static string Parse(string digest) } /// - /// FromBytes generates a digest from a byte. + /// CalculateSHA256DigestFromBytes generates a digest from a byte. /// /// /// - public static string FromBytes(byte[] content) + public static string CalculateSHA256DigestFromBytes(byte[] content) { using var sha256 = SHA256.Create(); var hash = sha256.ComputeHash(content); diff --git a/Oras/Interfaces/Registry/IRepositoryOption.cs b/Oras/Interfaces/Registry/IRepositoryOption.cs index 79bc1ce..f141752 100644 --- a/Oras/Interfaces/Registry/IRepositoryOption.cs +++ b/Oras/Interfaces/Registry/IRepositoryOption.cs @@ -39,14 +39,6 @@ public interface IRepositoryOption /// Reference: https://docs.docker.com/registry/spec/api/#tags /// public int TagListPageSize { get; set; } - - /// - /// MaxMetadataBytes specifies a limit on how many response bytes are allowed - /// in the server's response to the metadata APIs, such as catalog list, tag - /// list, and referrers list. - /// If less than or equal to zero, a default (currently 4MiB) is used. - /// - public long MaxMetadataBytes { get; set; } - + } } diff --git a/Oras/Remote/ErrorUtil.cs b/Oras/Remote/ErrorUtility.cs similarity index 95% rename from Oras/Remote/ErrorUtil.cs rename to Oras/Remote/ErrorUtility.cs index 04642aa..ef0a163 100644 --- a/Oras/Remote/ErrorUtil.cs +++ b/Oras/Remote/ErrorUtility.cs @@ -4,7 +4,7 @@ namespace Oras.Remote { - internal class ErrorUtil + internal class ErrorUtility { /// /// ParseErrorResponse parses the error returned by the remote registry. diff --git a/Oras/Remote/ManifestUtil.cs b/Oras/Remote/ManifestUtility.cs similarity index 87% rename from Oras/Remote/ManifestUtil.cs rename to Oras/Remote/ManifestUtility.cs index 4d6a0e7..af6a2b4 100644 --- a/Oras/Remote/ManifestUtil.cs +++ b/Oras/Remote/ManifestUtility.cs @@ -4,7 +4,7 @@ namespace Oras.Remote { - internal static class ManifestUtil + internal static class ManifestUtility { internal static string[] DefaultManifestMediaTypes = new[] { @@ -20,7 +20,7 @@ internal static class ManifestUtil /// /// /// - public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) + internal static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) { if (manifestMediaTypes == null || manifestMediaTypes.Length == 0) { @@ -40,7 +40,7 @@ public static bool IsManifest(string[] manifestMediaTypes, Descriptor desc) /// /// /// - public static string ManifestAcceptHeader(string[] manifestMediaTypes) + internal static string ManifestAcceptHeader(string[] manifestMediaTypes) { if (manifestMediaTypes == null || manifestMediaTypes.Length == 0) { diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index d2e2055..22a6c33 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Net.Http; using System.Text.RegularExpressions; +using Oras.Content; +using static Oras.Content.DigestUtility; namespace Oras.Remote { @@ -59,7 +61,7 @@ public RemoteReference ParseReference(string artifact) } (var registry, var path) = (parts[0], parts[1]); bool isTag = false; - string repository = String.Empty; + string repository; string reference = String.Empty; if (path.IndexOf('@') is var index && index != -1) @@ -88,30 +90,30 @@ public RemoteReference ParseReference(string artifact) // empty `reference`; Valid Form D repository = path; } - var refObj = new RemoteReference + var remoteReference = new RemoteReference { Registry = registry, Repository = repository, Reference = reference }; - refObj.ValidateRegistry(); - refObj.ValidateRepository(); + remoteReference.ValidateRegistry(); + remoteReference.ValidateRepository(); if (reference.Length == 0) { - return refObj; + return remoteReference; } if (isTag) { - refObj.ValidateReferenceAsTag(); + remoteReference.ValidateReferenceAsTag(); } else { - refObj.ValidateReferenceAsDigest(); + remoteReference.ValidateReferenceAsDigest(); } - return refObj; + return remoteReference; } /// @@ -144,10 +146,12 @@ public void ValidateRepository() /// public void ValidateRegistry() { - if (!Uri.IsWellFormedUriString("dummy://" + this.Registry, UriKind.Absolute)) + var url = "dummy://" + this.Registry; + if (!Uri.IsWellFormedUriString(url, UriKind.Absolute) || new Uri(url).Authority != Registry) { throw new InvalidReferenceException("Invalid Registry"); }; + } public void ValidateReferenceAsTag() @@ -164,7 +168,7 @@ public void ValidateReferenceAsTag() /// public void ValidateReference() { - if (Reference.Length == 0) + if (string.IsNullOrEmpty(Reference)) { return; } @@ -231,13 +235,14 @@ internal static void VerifyContentDigest(HttpResponseMessage resp, string refDig } } + /// + /// ParseDigest parses the digest from the string. + /// + /// + /// public static string ParseDigest(string digestStr) { - if (!Regex.IsMatch(digestStr, digestRegexp)) - { - throw new InvalidReferenceException($"invalid reference format: {digestStr}"); - } - return digestStr; + return ParseDigest(digestStr); } } } diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 74b8eda..b4ab75b 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -1,12 +1,17 @@ -using Oras.Exceptions; +using System.Collections.Generic; +using Oras.Exceptions; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Oras.Interfaces.Registry; +using System; +using System.Text.Json; +using static System.Web.HttpUtility; namespace Oras.Remote { - public class Registry + public class Registry : IRepositoryOption { public HttpClient HttpClient { get; set; } @@ -14,7 +19,6 @@ public class Registry public bool PlainHTTP { get; set; } public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } - public long MaxMetadataBytes { get; set; } /// /// Client returns an HTTP client used to access the remote repository. @@ -54,8 +58,85 @@ public async Task PingAsync(CancellationToken cancellationToken) case HttpStatusCode.NotFound: throw new NotFoundException($"Repository {RemoteReference} not found"); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } } + + /// + /// ListRepositoriesAsync returns a list of repositories from the remote registry. + /// + /// + /// + /// + /// + public async Task ListRepositoriesAsync(string last, Action fn, CancellationToken cancellationToken) + { + try + { + var url = URLUtiliity.BuildRegistryCatalogURL(PlainHTTP, RemoteReference); + while (true) + { + url = await RepositoryPageAsync(last, fn, url, cancellationToken); + last = ""; + } + } + catch (Utils.NoLinkHeaderException) + { + return; + } + } + + /// + /// RepositoryPageAsync returns a returns a single page of repositories list with the next link + /// + /// + /// + /// + /// + /// + private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) + { + if (PlainHTTP) + { + if (!url.Contains("http")) + { + url = "http://" + url; + } + } + else + { + if (!url.Contains("https")) + { + url = "https://" + url; + } + } + var uriBuilder = new UriBuilder(url); + var query = ParseQueryString(uriBuilder.Query); + if (TagListPageSize > 0 || last != "") + { + if (TagListPageSize > 0) + { + query["n"] = TagListPageSize.ToString(); + + + } + if (last != "") + { + query["last"] = last; + } + } + + uriBuilder.Query = query.ToString(); + var response = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); + if (response.StatusCode != HttpStatusCode.OK) + { + throw await ErrorUtility.ParseErrorResponse(response); + + } + var data = await response.Content.ReadAsStringAsync(); + var repositories = JsonSerializer.Deserialize(data); + fn(repositories); + return Utils.ParseLink(response); + } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index e20c6c8..e2eb13a 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -1,7 +1,7 @@ -using Oras.Exceptions; +using Oras.Content; +using Oras.Exceptions; using Oras.Interfaces.Registry; using Oras.Models; -using Oras.Utils; using System; using System.Collections.Generic; using System.IO; @@ -22,7 +22,6 @@ public class RepositoryOption : IRepositoryOption public bool PlainHTTP { get; set; } public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } - public long MaxMetadataBytes { get; set; } } /// /// Repository is an HTTP client to a remote repository @@ -68,14 +67,6 @@ public class Repository : IRepository, IRepositoryOption /// public int TagListPageSize { get; set; } - /// - /// MaxMetadataBytes specifies a limit on how many response bytes are allowed - /// in the server's response to the metadata APIs, such as catalog list, tag - /// list, and referrers list. - /// If less than or equal to zero, a default (currently 4MiB) is used. - /// - public long MaxMetadataBytes { get; set; } - /// /// dockerContentDigestHeader - The Docker-Content-Digest header, if present /// on the response, returns the canonical digest of the uploaded blob. @@ -112,8 +103,6 @@ public Repository(RemoteReference @ref, IRepositoryOption option) PlainHTTP = option.PlainHTTP; ManifestMediaTypes = option.ManifestMediaTypes; TagListPageSize = option.TagListPageSize; - MaxMetadataBytes = option.MaxMetadataBytes; - } /// @@ -141,7 +130,7 @@ private HttpClient Client() /// private IBlobStore blobStore(Descriptor desc) { - if (ManifestUtil.IsManifest(ManifestMediaTypes, desc)) + if (ManifestUtility.IsManifest(ManifestMediaTypes, desc)) { return Manifests(); } @@ -333,7 +322,7 @@ private async Task TagsPageAsync(string last, Action fn, strin var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } var data = await resp.Content.ReadAsStringAsync(); @@ -373,7 +362,7 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } } @@ -554,7 +543,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel case HttpStatusCode.NotFound: throw new NotFoundException($"digest {target.Digest} not found"); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } var mediaType = resp.Content.Headers?.ContentType.MediaType; if (mediaType != target.MediaType) @@ -624,7 +613,7 @@ private async Task PushAsync(Descriptor expected, Stream stream, string referenc var resp = await client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } RemoteReference.VerifyContentDigest(resp, expected.Digest); } @@ -634,14 +623,14 @@ public async Task ResolveAsync(string reference, CancellationToken c var refObj = Repository.ParseReference(reference); var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Head, url); - req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); var res = await Repository.HttpClient.SendAsync(req, cancellationToken); return res.StatusCode switch { - HttpStatusCode.OK => GenerateDescriptor(res, refObj, req.Method), + HttpStatusCode.OK => await GenerateDescriptor(res, refObj, req.Method), HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), - _ => throw await ErrorUtil.ParseErrorResponse(res) + _ => throw await ErrorUtility.ParseErrorResponse(res) }; } @@ -653,7 +642,7 @@ public async Task ResolveAsync(string reference, CancellationToken c /// /// /// - public Descriptor GenerateDescriptor(HttpResponseMessage res, RemoteReference @ref, HttpMethod httpMethod) + public async Task GenerateDescriptor(HttpResponseMessage res, RemoteReference @ref, HttpMethod httpMethod) { string mediaType; try @@ -721,7 +710,7 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, RemoteReference @r string calculatedDigest; try { - calculatedDigest = calculateDigestFromResponse(res, Repository.MaxMetadataBytes); + calculatedDigest = await CalculateDigestFromResponse(res); } catch (Exception e) { @@ -753,19 +742,10 @@ public Descriptor GenerateDescriptor(HttpResponseMessage res, RemoteReference @r /// taking care not to destroy it in the process /// /// - /// - private string calculateDigestFromResponse(HttpResponseMessage res, long maxMetadataBytes) + private async Task CalculateDigestFromResponse(HttpResponseMessage res) { - byte[] content = null; - try - { - content = Utils.LimitReader(res.Content, maxMetadataBytes); - } - catch (Exception ex) - { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: failed to read response body: {ex.Message}"); - } - return DigestUtility.FromBytes(content); + var bytes = await res.Content.ReadAsByteArrayAsync(); + return DigestUtility.CalculateSHA256DigestFromBytes(bytes); } /// @@ -791,7 +771,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var refObj = Repository.ParseReference(reference); var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Headers.Add("Accept", ManifestUtil.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { @@ -803,13 +783,13 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT } else { - desc = GenerateDescriptor(resp, refObj, HttpMethod.Get); + desc = await GenerateDescriptor(resp, refObj, HttpMethod.Get); } return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } } @@ -876,7 +856,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } } @@ -924,7 +904,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok var reqPort = resp.RequestMessage.RequestUri.Port; if (resp.StatusCode != HttpStatusCode.Accepted) { - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } string location; @@ -978,7 +958,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok resp = await Repository.HttpClient.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } return; @@ -1001,7 +981,7 @@ public async Task ResolveAsync(string reference, CancellationToken c { HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), HttpStatusCode.NotFound => throw new NotFoundException($"{refObj.Reference}: not found"), - _ => throw await ErrorUtil.ParseErrorResponse(resp) + _ => throw await ErrorUtility.ParseErrorResponse(resp) }; } @@ -1047,7 +1027,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT case HttpStatusCode.NotFound: throw new NotFoundException(); default: - throw await ErrorUtil.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp); } } diff --git a/Oras/Remote/URLUtiliity.cs b/Oras/Remote/URLUtiliity.cs index cc3b26b..45f5b08 100644 --- a/Oras/Remote/URLUtiliity.cs +++ b/Oras/Remote/URLUtiliity.cs @@ -24,11 +24,11 @@ internal static string BuildScheme(bool plainHTTP) /// Reference: https://docs.docker.com/registry/spec/api/#base /// /// - /// + /// /// - internal static string BuildRegistryBaseURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRegistryBaseURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/"; + return $"{BuildScheme(plainHTTP)}://{reference.Host()}/v2/"; } /// @@ -37,11 +37,11 @@ internal static string BuildRegistryBaseURL(bool plainHTTP, RemoteReference @ref /// Reference: https://docs.docker.com/registry/spec/api/#catalog /// /// - /// + /// /// - internal static string BuildRegistryCatalogURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRegistryCatalogURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/_catalog"; + return $"{BuildScheme(plainHTTP)}://{reference.Host()}/v2/_catalog"; } /// @@ -49,11 +49,11 @@ internal static string BuildRegistryCatalogURL(bool plainHTTP, RemoteReference @ /// Format: :///v2/ /// /// - /// + /// /// - internal static string BuildRepositoryBaseURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRepositoryBaseURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/{@ref.Repository}"; + return $"{BuildScheme(plainHTTP)}://{reference.Host()}/v2/{reference.Repository}"; } /// @@ -62,11 +62,11 @@ internal static string BuildRepositoryBaseURL(bool plainHTTP, RemoteReference @r /// Reference: https://docs.docker.com/registry/spec/api/#tags /// /// - /// + /// /// - internal static string BuildRepositoryTagListURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRepositoryTagListURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildScheme(plainHTTP)}://{@ref.Host()}/v2/{@ref.Repository}/tags/list"; + return $"{BuildRepositoryBaseURL(plainHTTP, reference)}/tags/list"; } /// @@ -75,11 +75,11 @@ internal static string BuildRepositoryTagListURL(bool plainHTTP, RemoteReference /// Reference: https://docs.docker.com/registry/spec/api/#manifest /// /// - /// + /// /// - internal static string BuildRepositoryManifestURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRepositoryManifestURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/manifests/{@ref.Reference}"; + return $"{BuildRepositoryBaseURL(plainHTTP, reference)}/manifests/{reference.Reference}"; } /// @@ -88,11 +88,11 @@ internal static string BuildRepositoryManifestURL(bool plainHTTP, RemoteReferenc /// Reference: https://docs.docker.com/registry/spec/api/#blob /// /// - /// + /// /// - internal static string BuildRepositoryBlobURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRepositoryBlobURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/blobs/{@ref.Reference}"; + return $"{BuildRepositoryBaseURL(plainHTTP, reference)}/blobs/{reference.Reference}"; } /// @@ -102,11 +102,11 @@ internal static string BuildRepositoryBlobURL(bool plainHTTP, RemoteReference @r /// /// - /// + /// /// - internal static string BuildRepositoryBlobUploadURL(bool plainHTTP, RemoteReference @ref) + internal static string BuildRepositoryBlobUploadURL(bool plainHTTP, RemoteReference reference) { - return $"{BuildRepositoryBaseURL(plainHTTP, @ref)}/blobs/uploads/"; + return $"{BuildRepositoryBaseURL(plainHTTP, reference)}/blobs/uploads/"; } } diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index f1337fd..721815a 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -6,13 +6,6 @@ namespace Oras.Remote { internal class Utils { - /// - /// defaultMaxMetadataBytes specifies the default limit on how many response - /// bytes are allowed in the server's response to the metadata APIs. - /// See also: Repository.MaxMetadataBytes - /// - const long defaultMaxMetadataBytes = 4 * 1024 * 1024; // 4 MiB - /// /// ParseLink returns the URL of the response's "Link" header, if present. /// @@ -20,7 +13,7 @@ internal class Utils /// public static string ParseLink(HttpResponseMessage resp) { - var link = String.Empty; + string link; if (resp.Headers.TryGetValues("Link", out var values)) { link = values.FirstOrDefault(); @@ -30,12 +23,11 @@ public static string ParseLink(HttpResponseMessage resp) throw new NoLinkHeaderException(); } - if (link[0] != '<') { throw new Exception($"invalid next link {link}: missing '<"); } - if (link.IndexOf('>') is var index && index < -1) + if (link.IndexOf('>') is var index && index == -1) { throw new Exception($"invalid next link {link}: missing '>'"); } @@ -56,36 +48,12 @@ public static string ParseLink(HttpResponseMessage resp) return resolvedUri.AbsoluteUri; } - - /// - /// LimitReader ensures that the read byte does not exceed n - /// bytes. if n is less than or equal to zero, defaultMaxMetadataBytes is used. - /// - /// - /// - /// - /// - public static byte[] LimitReader(HttpContent content, long n) - { - if (n <= 0) - { - n = defaultMaxMetadataBytes; - } - - var bytes = content.ReadAsByteArrayAsync().Result; - - if (bytes.Length > n) - { - throw new Exception($"response body exceeds the limit of {n} bytes"); - } - - return bytes; - } + /// /// NoLinkHeaderException is thrown when a link header is missing. /// - public class NoLinkHeaderException : Exception + internal class NoLinkHeaderException : Exception { public NoLinkHeaderException() { From f19f348738d0e2a20452ec18a315747b465922f8 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 17:49:24 +0100 Subject: [PATCH 54/77] =?UTF-8?q?fix:=20flash=20mode=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Samson Amaugo --- Oras/Remote/Reference.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 22a6c33..54f6782 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -242,7 +242,7 @@ internal static void VerifyContentDigest(HttpResponseMessage resp, string refDig /// public static string ParseDigest(string digestStr) { - return ParseDigest(digestStr); + return DigestUtility.Parse(digestStr); } } } From fb25cb65dfc7d09259c19a110a745c849c290833 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 19:04:15 +0100 Subject: [PATCH 55/77] added more test Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RegistryTest.cs | 150 ++++++++++++++++++++++++++ Oras/Remote/Registry.cs | 10 ++ 2 files changed, 160 insertions(+) create mode 100644 Oras.Tests/RemoteTest/RegistryTest.cs diff --git a/Oras.Tests/RemoteTest/RegistryTest.cs b/Oras.Tests/RemoteTest/RegistryTest.cs new file mode 100644 index 0000000..cd89a1b --- /dev/null +++ b/Oras.Tests/RemoteTest/RegistryTest.cs @@ -0,0 +1,150 @@ +using Moq.Protected; +using Moq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Xunit; +using Oras.Remote; +using System.Text.RegularExpressions; + +namespace Oras.Tests.RemoteTest +{ + public class RegistryTest + { + + public static HttpClient CustomClient(Func func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).ReturnsAsync(func); + return new HttpClient(moqHandler.Object); + } + + /// + /// TestRegistry_PingAsync tests the PingAsync method of the Registry class. + /// + /// + [Fact] + public async Task TestRegistry_PingAsync() + { + var V2Implemented = true; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + + if (req.Method != HttpMethod.Get && req.RequestUri.AbsolutePath == $"/v2/") + { + res.StatusCode = HttpStatusCode.NotFound; + return res; + } + + if (V2Implemented) + { + res.StatusCode = HttpStatusCode.OK; + return res; + } + else + { + res.StatusCode = HttpStatusCode.NotFound; + return res; + } + }; + var registry = new Oras.Remote.Registry("localhost:5000"); + registry.PlainHTTP = true; + registry.HttpClient = CustomClient(func); + var cancellationToken = new CancellationToken(); + await registry.PingAsync(cancellationToken); + V2Implemented = false; + await Assert.ThrowsAnyAsync( + async () => await registry.PingAsync(cancellationToken)); + } + + /// + /// TestRegistry_ListRepositoriesAsync tests the ListRepositoriesAsync method of the Registry class. + /// + /// + /// + [Fact] + public async Task TestRegistry_ListRepositoriesAsync() + { + var repoSet = new List>() + { + new() {"the", "quick", "brown", "fox"}, + new() {"jumps", "over", "the", "lazy"}, + new() {"dog"} + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get || + req.RequestUri.AbsolutePath != "/v2/_catalog" + ) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + var q = req.RequestUri.Query; + try + { + var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); + if (n != 4) throw new Exception(); + } + catch (Exception e) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + var repos = new List(); + var serverUrl = "http://localhost:5000"; + var matched = Regex.Match(q, @"(?<=test=)\w+").Value; + switch (matched) + { + case "foo": + repos = repoSet[1]; + res.Headers.Add("Link", $"<{serverUrl}/v2/_catalog?n=4&test=bar>; rel=\"next\""); + break; + case "bar": + repos = repoSet[2]; + break; + default: + repos = repoSet[0]; + res.Headers.Add("Link", $"; rel=\"next\""); + break; + } + res.Content = new StringContent(JsonSerializer.Serialize(repos)); + return res; + + }; + + var registry = new Oras.Remote.Registry("localhost:5000"); + registry.PlainHTTP = true; + registry.HttpClient = CustomClient(func); + var cancellationToken = new CancellationToken(); + registry.TagListPageSize = 4; + + + + var index = 0; + await registry.ListRepositoriesAsync("", async (string[] got) => + { + if (index > 2) + { + throw new Exception($"Error out of range: {index}"); + } + + var repos = repoSet[index]; + index++; + Assert.Equal(got, repos); + }, cancellationToken); + } + } +} diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index b4ab75b..be8a03c 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -37,6 +37,16 @@ private HttpClient Client() return HttpClient; } + public Registry(string name) + { + var reference = new RemoteReference + { + Registry = name, + }; + reference.ValidateRegistry(); + RemoteReference = reference; + } + /// /// PingAsync checks whether or not the registry implement Docker Registry API V2 or /// OCI Distribution Specification. From 518c763215856b9298592ce8e64ed03299fee6cc Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Thu, 25 May 2023 19:05:56 +0100 Subject: [PATCH 56/77] formatted code Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RegistryTest.cs | 14 ++++---------- Oras.Tests/RemoteTest/RemoteTest.cs | 3 +-- Oras/Content/DigestUtility.cs | 3 +-- Oras/Interfaces/Registry/IRepositoryOption.cs | 2 +- Oras/Models/Descriptor.cs | 5 ++--- Oras/Remote/Reference.cs | 7 +++---- Oras/Remote/Registry.cs | 9 ++++----- Oras/Remote/Utils.cs | 2 +- 8 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Oras.Tests/RemoteTest/RegistryTest.cs b/Oras.Tests/RemoteTest/RegistryTest.cs index cd89a1b..79ed7a1 100644 --- a/Oras.Tests/RemoteTest/RegistryTest.cs +++ b/Oras.Tests/RemoteTest/RegistryTest.cs @@ -1,15 +1,9 @@ -using Moq.Protected; -using Moq; -using System; -using System.Collections.Generic; -using System.Linq; +using Moq; +using Moq.Protected; using System.Net; -using System.Text; using System.Text.Json; -using System.Threading.Tasks; -using Xunit; -using Oras.Remote; using System.Text.RegularExpressions; +using Xunit; namespace Oras.Tests.RemoteTest { @@ -131,7 +125,7 @@ public async Task TestRegistry_ListRepositoriesAsync() var cancellationToken = new CancellationToken(); registry.TagListPageSize = 4; - + var index = 0; await registry.ListRepositoriesAsync("", async (string[] got) => diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RemoteTest.cs index 5b660d1..52d096c 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RemoteTest.cs @@ -14,7 +14,6 @@ using System.Web; using Xunit; using static Oras.Content.Content; -using static System.Runtime.InteropServices.JavaScript.JSType; namespace Oras.Tests.RemoteTest { @@ -121,7 +120,7 @@ public bool AreDescriptorsEqual(Descriptor a, Descriptor b) { return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; } - + public static HttpClient CustomClient(Func func) { var moqHandler = new Mock(); diff --git a/Oras/Content/DigestUtility.cs b/Oras/Content/DigestUtility.cs index 7113a70..153ed17 100644 --- a/Oras/Content/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -1,5 +1,4 @@ using Oras.Exceptions; -using Oras.Remote; using System; using System.Security.Cryptography; using System.Text.RegularExpressions; @@ -13,7 +12,7 @@ internal class DigestUtility /// digestRegexp checks the digest. /// public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; - + /// /// Parse verifies the digest header and throws an exception if it is invalid. /// diff --git a/Oras/Interfaces/Registry/IRepositoryOption.cs b/Oras/Interfaces/Registry/IRepositoryOption.cs index f141752..80b4693 100644 --- a/Oras/Interfaces/Registry/IRepositoryOption.cs +++ b/Oras/Interfaces/Registry/IRepositoryOption.cs @@ -39,6 +39,6 @@ public interface IRepositoryOption /// Reference: https://docs.docker.com/registry/spec/api/#tags /// public int TagListPageSize { get; set; } - + } } diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index 82c7ede..c6e383a 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Text.Json.Serialization; @@ -37,7 +36,7 @@ public class Descriptor [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string ArtifactType { get; set; } - internal MinimumDescriptor GetMinimum() + internal MinimumDescriptor GetMinimum() { return new MinimumDescriptor { diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/Reference.cs index 54f6782..64e1f84 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/Reference.cs @@ -1,10 +1,9 @@ -using Oras.Exceptions; +using Oras.Content; +using Oras.Exceptions; using System; using System.Linq; using System.Net.Http; using System.Text.RegularExpressions; -using Oras.Content; -using static Oras.Content.DigestUtility; namespace Oras.Remote { @@ -151,7 +150,7 @@ public void ValidateRegistry() { throw new InvalidReferenceException("Invalid Registry"); }; - + } public void ValidateReferenceAsTag() diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index be8a03c..312bfdd 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -1,12 +1,11 @@ -using System.Collections.Generic; -using Oras.Exceptions; +using Oras.Exceptions; +using Oras.Interfaces.Registry; +using System; using System.Net; using System.Net.Http; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Oras.Interfaces.Registry; -using System; -using System.Text.Json; using static System.Web.HttpUtility; namespace Oras.Remote diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/Utils.cs index 721815a..883f1c7 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/Utils.cs @@ -48,7 +48,7 @@ public static string ParseLink(HttpResponseMessage resp) return resolvedUri.AbsoluteUri; } - + /// /// NoLinkHeaderException is thrown when a link header is missing. From f13dbb52a1c3eed6e15efe57884257ce417aebd5 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 26 May 2023 06:48:18 +0100 Subject: [PATCH 57/77] made some changes Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index e2eb13a..4c91c33 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -28,13 +28,6 @@ public class RepositoryOption : IRepositoryOption /// public class Repository : IRepository, IRepositoryOption { - /// - /// bytes are allowed in the server's response to the metadata APIs. - /// defaultMaxMetadataBytes specifies the default limit on how many response - /// See also: Repository.MaxMetadataBytes - /// - public long defaultMaxMetaBytes = 4 * 1024 * 1024; //4 Mib - /// /// HttpClient is the underlying HTTP client used to access the remote registry. /// @@ -638,11 +631,11 @@ public async Task ResolveAsync(string reference, CancellationToken c /// GenerateDescriptor returns a descriptor generated from the response. /// /// - /// + /// /// /// /// - public async Task GenerateDescriptor(HttpResponseMessage res, RemoteReference @ref, HttpMethod httpMethod) + public async Task GenerateDescriptor(HttpResponseMessage res, RemoteReference reference, HttpMethod httpMethod) { string mediaType; try @@ -666,14 +659,15 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote string refDigest = string.Empty; try { - refDigest = @ref.Digest(); + refDigest = reference.Digest(); } catch (Exception) { } + // 4. Validate Server Digest (if present) - res.Content.Headers.TryGetValues("Docker-Content-Digest", out var serverHeaderDigest); + var serverHeaderDigest = res.Content.Headers.GetValues("Docker-Content-Digest"); var serverDigest = serverHeaderDigest.First(); if (!string.IsNullOrEmpty(serverDigest)) { From 8a2177955a180c6308d6923de7e58ad723b962c4 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 26 May 2023 07:03:48 +0100 Subject: [PATCH 58/77] made some changes Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RegistryTest.cs | 6 ++-- Oras/Interfaces/Registry/IRegistry.cs | 41 +++++++++++++++++++++++++++ Oras/Remote/Registry.cs | 23 +++++++++++++-- Oras/Remote/Repository.cs | 8 +++--- 4 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 Oras/Interfaces/Registry/IRegistry.cs diff --git a/Oras.Tests/RemoteTest/RegistryTest.cs b/Oras.Tests/RemoteTest/RegistryTest.cs index 79ed7a1..1a84a77 100644 --- a/Oras.Tests/RemoteTest/RegistryTest.cs +++ b/Oras.Tests/RemoteTest/RegistryTest.cs @@ -62,12 +62,12 @@ await Assert.ThrowsAnyAsync( } /// - /// TestRegistry_ListRepositoriesAsync tests the ListRepositoriesAsync method of the Registry class. + /// TestRegistry_Repositories tests the ListRepositoriesAsync method of the Registry class. /// /// /// [Fact] - public async Task TestRegistry_ListRepositoriesAsync() + public async Task TestRegistry_Repositories() { var repoSet = new List>() { @@ -128,7 +128,7 @@ public async Task TestRegistry_ListRepositoriesAsync() var index = 0; - await registry.ListRepositoriesAsync("", async (string[] got) => + await registry.Repositories("", async (string[] got) => { if (index > 2) { diff --git a/Oras/Interfaces/Registry/IRegistry.cs b/Oras/Interfaces/Registry/IRegistry.cs new file mode 100644 index 0000000..9dd2eb7 --- /dev/null +++ b/Oras/Interfaces/Registry/IRegistry.cs @@ -0,0 +1,41 @@ +using Oras.Remote; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using System.Threading; + +namespace Oras.Interfaces.Registry +{ + public interface IRegistry + { + /// + /// Repository returns a repository reference by the given name. + /// + /// + /// + /// + Task Repository(string name, CancellationToken cancellationToken); + + /// + /// Repositories lists the name of repositories available in the registry. + /// Since the returned repositories may be paginated by the underlying + /// implementation, a function should be passed in to process the paginated + /// repository list. + /// `last` argument is the `last` parameter when invoking the catalog API. + /// If `last` is NOT empty, the entries in the response start after the + /// repo specified by `last`. Otherwise, the response starts from the top + /// of the Repositories list. + /// Note: When implemented by a remote registry, the catalog API is called. + /// However, not all registries supports pagination or conforms the + /// specification. + /// Reference: https://docs.docker.com/registry/spec/api/#catalog + /// See also `Repositories()` in this package. + /// + /// + /// + /// + /// + Task Repositories(string last, Action fn, CancellationToken cancellationToken); + } +} diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 312bfdd..1d8d936 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -10,7 +10,7 @@ namespace Oras.Remote { - public class Registry : IRepositoryOption + public class Registry : IRegistry { public HttpClient HttpClient { get; set; } @@ -72,13 +72,30 @@ public async Task PingAsync(CancellationToken cancellationToken) } /// - /// ListRepositoriesAsync returns a list of repositories from the remote registry. + /// Repository returns a repository object for the given repository name. + /// + /// + /// + /// + public async Task Repository(string name, CancellationToken cancellationToken) + { + var reference = new RemoteReference + { + Registry = RemoteReference.Registry, + Repository = name, + }; + return new Repository(reference, new RepositoryOption()); + } + + + /// + /// Repositories returns a list of repositories from the remote registry. /// /// /// /// /// - public async Task ListRepositoriesAsync(string last, Action fn, CancellationToken cancellationToken) + public async Task Repositories(string last, Action fn, CancellationToken cancellationToken) { try { diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 4c91c33..7ea49c7 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -86,13 +86,13 @@ public Repository(string reference) /// to multiple Repositories. To handle this we explicitly copy only the /// fields that we want to reproduce. /// - /// + /// /// - public Repository(RemoteReference @ref, IRepositoryOption option) + public Repository(RemoteReference reference, IRepositoryOption option) { - @ref.ValidateRepository(); + reference.ValidateRepository(); HttpClient = option.HttpClient; - RemoteReference = @ref; + RemoteReference = reference; PlainHTTP = option.PlainHTTP; ManifestMediaTypes = option.ManifestMediaTypes; TagListPageSize = option.TagListPageSize; From d40a0bfa7ae8378b8deee7c50f20f8400a3fa6d3 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Fri, 26 May 2023 18:41:32 +0100 Subject: [PATCH 59/77] made some changes Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RegistryTest.cs | 8 +- .../{RemoteTest.cs => RepositoryTest.cs} | 9 +- Oras/Content/DigestUtility.cs | 14 +- Oras/Interfaces/Registry/IRegistry.cs | 2 +- Oras/Memory/MemoryGraph.cs | 6 +- Oras/Memory/MemoryStorage.cs | 6 +- Oras/Models/Descriptor.cs | 2 +- Oras/Remote/{Utils.cs => LinkUtils.cs} | 29 ++- Oras/Remote/Registry.cs | 82 ++++--- .../{Reference.cs => RemoteReference.cs} | 55 +---- Oras/Remote/Repository.cs | 217 ++++++++---------- Oras/Remote/ResponseTypes.cs | 22 ++ 12 files changed, 216 insertions(+), 236 deletions(-) rename Oras.Tests/RemoteTest/{RemoteTest.cs => RepositoryTest.cs} (99%) rename Oras/Remote/{Utils.cs => LinkUtils.cs} (71%) rename Oras/Remote/{Reference.cs => RemoteReference.cs} (74%) create mode 100644 Oras/Remote/ResponseTypes.cs diff --git a/Oras.Tests/RemoteTest/RegistryTest.cs b/Oras.Tests/RemoteTest/RegistryTest.cs index 1a84a77..85870b5 100644 --- a/Oras.Tests/RemoteTest/RegistryTest.cs +++ b/Oras.Tests/RemoteTest/RegistryTest.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text.Json; using System.Text.RegularExpressions; +using Oras.Remote; using Xunit; namespace Oras.Tests.RemoteTest @@ -114,7 +115,12 @@ public async Task TestRegistry_Repositories() res.Headers.Add("Link", $"; rel=\"next\""); break; } - res.Content = new StringContent(JsonSerializer.Serialize(repos)); + + var repositoryList = new ResponseTypes.RepositoryList + { + Repositories = repos.ToArray() + }; + res.Content = new StringContent(JsonSerializer.Serialize(repositoryList)); return res; }; diff --git a/Oras.Tests/RemoteTest/RemoteTest.cs b/Oras.Tests/RemoteTest/RepositoryTest.cs similarity index 99% rename from Oras.Tests/RemoteTest/RemoteTest.cs rename to Oras.Tests/RemoteTest/RepositoryTest.cs index 52d096c..9cda773 100644 --- a/Oras.Tests/RemoteTest/RemoteTest.cs +++ b/Oras.Tests/RemoteTest/RepositoryTest.cs @@ -17,7 +17,7 @@ namespace Oras.Tests.RemoteTest { - public class RemoteTest + public class RepositoryTest { public struct TestIOStruct { @@ -772,7 +772,12 @@ public async Task Repository_TagsAsync() res.Headers.Add("Link", $"; rel=\"next\""); break; } - res.Content = new StringContent(JsonSerializer.Serialize(tags)); + + var listOfTags = new ResponseTypes.TagList + { + Tags = tags.ToArray() + }; + res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); return res; }; diff --git a/Oras/Content/DigestUtility.cs b/Oras/Content/DigestUtility.cs index 153ed17..ec0407a 100644 --- a/Oras/Content/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -5,34 +5,34 @@ namespace Oras.Content { - internal class DigestUtility + internal static class DigestUtility { /// /// digestRegexp checks the digest. /// - public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; + private static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; /// - /// Parse verifies the digest header and throws an exception if it is invalid. + /// ParseDigest verifies the digest header and throws an exception if it is invalid. /// /// - public static string Parse(string digest) + internal static string ParseDigest(string digest) { if (!Regex.IsMatch(digest, digestRegexp)) { - throw new InvalidReferenceException($"invalid reference format: {digest}"); + throw new InvalidReferenceException($"Invalid digest: {digest}"); } return digest; } /// - /// CalculateSHA256DigestFromBytes generates a digest from a byte. + /// CalculateSHA256DigestFromBytes generates a SHA256 digest from a byte array. /// /// /// - public static string CalculateSHA256DigestFromBytes(byte[] content) + internal static string CalculateSHA256DigestFromBytes(byte[] content) { using var sha256 = SHA256.Create(); var hash = sha256.ComputeHash(content); diff --git a/Oras/Interfaces/Registry/IRegistry.cs b/Oras/Interfaces/Registry/IRegistry.cs index 9dd2eb7..fc87ef7 100644 --- a/Oras/Interfaces/Registry/IRegistry.cs +++ b/Oras/Interfaces/Registry/IRegistry.cs @@ -15,7 +15,7 @@ public interface IRegistry /// /// /// - Task Repository(string name, CancellationToken cancellationToken); + Task Repository(string name, CancellationToken cancellationToken); /// /// Repositories lists the name of repositories available in the registry. diff --git a/Oras/Memory/MemoryGraph.cs b/Oras/Memory/MemoryGraph.cs index c9cd9af..0a2c178 100644 --- a/Oras/Memory/MemoryGraph.cs +++ b/Oras/Memory/MemoryGraph.cs @@ -29,7 +29,7 @@ internal async Task IndexAsync(IFetcher fetcher, Descriptor node, CancellationTo /// internal async Task> PredecessorsAsync(Descriptor node, CancellationToken cancellationToken) { - var key = node.GetMinimum(); + var key = node.GetMinimumDescriptor(); if (!this._predecessors.TryGetValue(key, out ConcurrentDictionary predecessors)) { return default; @@ -53,10 +53,10 @@ private void Index(Descriptor node, IList successors, CancellationTo return; } - var predecessorKey = node.GetMinimum(); + var predecessorKey = node.GetMinimumDescriptor(); foreach (var successor in successors) { - var successorKey = successor.GetMinimum(); + var successorKey = successor.GetMinimumDescriptor(); var predecessors = this._predecessors.GetOrAdd(successorKey, new ConcurrentDictionary()); predecessors.TryAdd(predecessorKey, node); } diff --git a/Oras/Memory/MemoryStorage.cs b/Oras/Memory/MemoryStorage.cs index b207051..f95e59f 100644 --- a/Oras/Memory/MemoryStorage.cs +++ b/Oras/Memory/MemoryStorage.cs @@ -15,7 +15,7 @@ internal class MemoryStorage : IStorage public Task ExistsAsync(Descriptor target, CancellationToken cancellationToken) { - var contentExist = _content.ContainsKey(target.GetMinimum()); + var contentExist = _content.ContainsKey(target.GetMinimumDescriptor()); return Task.FromResult(contentExist); } @@ -23,7 +23,7 @@ public Task ExistsAsync(Descriptor target, CancellationToken cancellationT public Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var contentExist = this._content.TryGetValue(target.GetMinimum(), out byte[] content); + var contentExist = this._content.TryGetValue(target.GetMinimumDescriptor(), out byte[] content); if (!contentExist) { throw new NotFoundException($"{target.Digest} : {target.MediaType}"); @@ -34,7 +34,7 @@ public Task FetchAsync(Descriptor target, CancellationToken cancellation public async Task PushAsync(Descriptor expected, Stream contentStream, CancellationToken cancellationToken = default) { - var key = expected.GetMinimum(); + var key = expected.GetMinimumDescriptor(); var contentExist = _content.TryGetValue(key, out byte[] _); if (contentExist) { diff --git a/Oras/Models/Descriptor.cs b/Oras/Models/Descriptor.cs index c6e383a..2ef549b 100644 --- a/Oras/Models/Descriptor.cs +++ b/Oras/Models/Descriptor.cs @@ -36,7 +36,7 @@ public class Descriptor [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string ArtifactType { get; set; } - internal MinimumDescriptor GetMinimum() + internal MinimumDescriptor GetMinimumDescriptor() { return new MinimumDescriptor { diff --git a/Oras/Remote/Utils.cs b/Oras/Remote/LinkUtils.cs similarity index 71% rename from Oras/Remote/Utils.cs rename to Oras/Remote/LinkUtils.cs index 883f1c7..17de88b 100644 --- a/Oras/Remote/Utils.cs +++ b/Oras/Remote/LinkUtils.cs @@ -4,14 +4,14 @@ namespace Oras.Remote { - internal class Utils + internal class LinkUtils { /// /// ParseLink returns the URL of the response's "Link" header, if present. /// /// /// - public static string ParseLink(HttpResponseMessage resp) + internal static string ParseLink(HttpResponseMessage resp) { string link; if (resp.Headers.TryGetValues("Link", out var values)) @@ -49,6 +49,31 @@ public static string ParseLink(HttpResponseMessage resp) return resolvedUri.AbsoluteUri; } + /// + /// ObtainUrl returns the link with a scheme and authority. + /// + /// + /// + /// + internal static string ObtainUrl(string url, bool plainHttp) + { + if (plainHttp) + { + if (!url.Contains("http")) + { + url = "http://" + url; + } + } + else + { + if (!url.Contains("https")) + { + url = "https://" + url; + } + } + + return url; + } /// /// NoLinkHeaderException is thrown when a link header is missing. diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 1d8d936..bc9c256 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Text.Json; using System.Threading; +using Oras.Remote; using System.Threading.Tasks; using static System.Web.HttpUtility; @@ -19,22 +20,7 @@ public class Registry : IRegistry public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } - /// - /// Client returns an HTTP client used to access the remote repository. - /// A default HTTP client is return if the client is not configured. - /// - /// - private HttpClient Client() - { - if (HttpClient is null) - { - var client = new HttpClient(); - client.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); - return client; - } - - return HttpClient; - } + public Registry(string name) { @@ -44,6 +30,8 @@ public Registry(string name) }; reference.ValidateRegistry(); RemoteReference = reference; + HttpClient = new HttpClient(); + HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); } /// @@ -59,7 +47,7 @@ public Registry(string name) public async Task PingAsync(CancellationToken cancellationToken) { var url = URLUtiliity.BuildRegistryBaseURL(PlainHTTP, RemoteReference); - var resp = await Client().GetAsync(url, cancellationToken); + using var resp = await HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -75,19 +63,39 @@ public async Task PingAsync(CancellationToken cancellationToken) /// Repository returns a repository object for the given repository name. /// /// - /// + /// + /// + public async Task Repository(string name, CancellationToken cancellationToken) + { + var reference = new RemoteReference + { + Registry = RemoteReference.Registry, + Repository = name, + }; + return new Repository(reference,HttpClient); + } + + + /// + /// Repository returns a repository object for the given repository name. + /// + /// + /// /// - public async Task Repository(string name, CancellationToken cancellationToken) + public async Task Repository(string name,HttpClient httpClient, CancellationToken cancellationToken) { var reference = new RemoteReference { Registry = RemoteReference.Registry, Repository = name, }; - return new Repository(reference, new RepositoryOption()); + HttpClient = httpClient; + HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); + + return new Repository(reference, HttpClient); } - - + + /// /// Repositories returns a list of repositories from the remote registry. /// @@ -106,7 +114,7 @@ public async Task Repositories(string last, Action fn, CancellationTok last = ""; } } - catch (Utils.NoLinkHeaderException) + catch (LinkUtils.NoLinkHeaderException) { return; } @@ -122,20 +130,8 @@ public async Task Repositories(string last, Action fn, CancellationTok /// private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - if (PlainHTTP) - { - if (!url.Contains("http")) - { - url = "http://" + url; - } - } - else - { - if (!url.Contains("https")) - { - url = "https://" + url; - } - } + + url = LinkUtils.ObtainUrl(url,PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -146,23 +142,25 @@ private async Task RepositoryPageAsync(string last, Action fn, } - if (last != "") + if (!string.IsNullOrEmpty(last)) { query["last"] = last; } } uriBuilder.Query = query.ToString(); - var response = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); + using var response = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); if (response.StatusCode != HttpStatusCode.OK) { throw await ErrorUtility.ParseErrorResponse(response); } var data = await response.Content.ReadAsStringAsync(); - var repositories = JsonSerializer.Deserialize(data); - fn(repositories); - return Utils.ParseLink(response); + var repositories = JsonSerializer.Deserialize(data); + fn(repositories.Repositories); + return LinkUtils.ParseLink(response); } + + } } diff --git a/Oras/Remote/Reference.cs b/Oras/Remote/RemoteReference.cs similarity index 74% rename from Oras/Remote/Reference.cs rename to Oras/Remote/RemoteReference.cs index 64e1f84..d079cc6 100644 --- a/Oras/Remote/Reference.cs +++ b/Oras/Remote/RemoteReference.cs @@ -38,20 +38,16 @@ public class RemoteReference /// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53 /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - public static string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; + private static string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; /// /// tagRegexp checks the tag name. /// The docker and OCI spec have the same regular expression. /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - public static string tagRegexp = @"^[\w][\w.-]{0,127}$"; + private static string tagRegexp = @"^[\w][\w.-]{0,127}$"; - /// - /// digestRegexp checks the digest. - /// - public static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; - public RemoteReference ParseReference(string artifact) + public static RemoteReference ParseReference(string artifact) { var parts = artifact.Split('/', 2); if (parts.Length == 1) @@ -99,7 +95,7 @@ public RemoteReference ParseReference(string artifact) remoteReference.ValidateRegistry(); remoteReference.ValidateRepository(); - if (reference.Length == 0) + if (string.IsNullOrEmpty(reference)) { return remoteReference; } @@ -120,10 +116,7 @@ public RemoteReference ParseReference(string artifact) /// public void ValidateReferenceAsDigest() { - if (!Regex.IsMatch(Reference, digestRegexp)) - { - throw new InvalidReferenceException($"invalid reference format: {Reference}"); - } + DigestUtility.ParseDigest(Reference); } @@ -205,43 +198,5 @@ public string Digest() return Reference; } - /// - /// VerifyContentDigest verifies the content digest of the artifact. - /// - /// - /// - /// - internal static void VerifyContentDigest(HttpResponseMessage resp, string refDigest) - { - var digestStr = resp.Content.Headers.GetValues("Docker-Content-Digest").FirstOrDefault(); - if (String.IsNullOrEmpty(digestStr)) - { - return; - } - - string contentDigest; - try - { - contentDigest = ParseDigest(digestStr); - } - catch (Exception) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); - } - if (contentDigest != refDigest) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {refDigest}"); - } - } - - /// - /// ParseDigest parses the digest from the string. - /// - /// - /// - public static string ParseDigest(string digestStr) - { - return DigestUtility.Parse(digestStr); - } } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 7ea49c7..608cda0 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -15,18 +15,11 @@ using static System.Web.HttpUtility; namespace Oras.Remote { - public class RepositoryOption : IRepositoryOption - { - public HttpClient HttpClient { get; set; } - public RemoteReference RemoteReference { get; set; } - public bool PlainHTTP { get; set; } - public string[] ManifestMediaTypes { get; set; } - public int TagListPageSize { get; set; } - } + /// /// Repository is an HTTP client to a remote repository /// - public class Repository : IRepository, IRepositoryOption + public class Repository : IRepository { /// /// HttpClient is the underlying HTTP client used to access the remote registry. @@ -60,14 +53,6 @@ public class Repository : IRepository, IRepositoryOption /// public int TagListPageSize { get; set; } - /// - /// dockerContentDigestHeader - The Docker-Content-Digest header, if present - /// on the response, returns the canonical digest of the uploaded blob. - /// See https://docs.docker.com/registry/spec/api/#digest-header - /// See https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-digests - /// - public const string DockerContentDigestHeader = "Docker-Content-Digest"; - /// /// Creates a client to the remote repository identified by a reference /// Example: localhost:5000/hello-world @@ -75,27 +60,20 @@ public class Repository : IRepository, IRepositoryOption /// public Repository(string reference) { - var refObj = new RemoteReference().ParseReference(reference); - RemoteReference = refObj; + RemoteReference = RemoteReference.ParseReference(reference); ; } /// - /// This constructor customizes the Properties using the values - /// from the RepositoryOptions. - /// RepositoryOptions contains unexported state that must not be copied - /// to multiple Repositories. To handle this we explicitly copy only the - /// fields that we want to reproduce. + /// This constructor customizes the HttpClient and sets the properties + /// using values from the parameter. /// /// - /// - public Repository(RemoteReference reference, IRepositoryOption option) + /// + public Repository(RemoteReference reference, HttpClient httpClient) { reference.ValidateRepository(); - HttpClient = option.HttpClient; + HttpClient = httpClient; RemoteReference = reference; - PlainHTTP = option.PlainHTTP; - ManifestMediaTypes = option.ManifestMediaTypes; - TagListPageSize = option.TagListPageSize; } /// @@ -227,6 +205,13 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { await blobStore(target).DeleteAsync(target, cancellationToken); } + + /// + /// TagsAsync returns a list of tags in a repository + /// + /// + /// + /// public async Task> TagsAsync(ITagLister repo, CancellationToken cancellationToken) { var res = new List(); @@ -264,7 +249,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken last = ""; } } - catch (Utils.NoLinkHeaderException) + catch (LinkUtils.NoLinkHeaderException) { return; } @@ -281,20 +266,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken /// private async Task TagsPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - if (PlainHTTP) - { - if (!url.Contains("http")) - { - url = "http://" + url; - } - } - else - { - if (!url.Contains("https")) - { - url = "https://" + url; - } - } + url = LinkUtils.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -319,9 +291,9 @@ private async Task TagsPageAsync(string last, Action fn, strin } var data = await resp.Content.ReadAsStringAsync(); - var tags = JsonSerializer.Deserialize(data); - fn(tags); - return Utils.ParseLink(resp); + var tagList = JsonSerializer.Deserialize(data); + fn(tagList.Tags); + return LinkUtils.ParseLink(resp); } /// @@ -336,15 +308,18 @@ private async Task TagsPageAsync(string last, Action fn, strin /// internal async Task DeleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) { - var refObj = RemoteReference; - refObj.Reference = target.Digest; - Func buildURL = URLUtiliity.BuildRepositoryBlobURL; + var remoteReference = RemoteReference; + remoteReference.Reference = target.Digest; + string url; if (isManifest) { - buildURL = URLUtiliity.BuildRepositoryManifestURL; + url = URLUtiliity.BuildRepositoryManifestURL(PlainHTTP, remoteReference); + } + else + { + url = URLUtiliity.BuildRepositoryBlobURL(PlainHTTP, remoteReference); } - var url = buildURL(PlainHTTP, refObj); var resp = await HttpClient.DeleteAsync(url, cancellationToken); switch (resp.StatusCode) @@ -368,10 +343,11 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation /// /// /// - private void VerifyContentDigest(HttpResponseMessage resp, string expected) + internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) { - resp.Content.Headers.TryGetValues(DockerContentDigestHeader, out var digestStr); - if (digestStr == null || !digestStr.Any()) + if (resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues) is var gotValues && !gotValues) return; + var digestStr = digestValues.FirstOrDefault(); + if (string.IsNullOrEmpty(digestStr)) { return; } @@ -379,22 +355,16 @@ private void VerifyContentDigest(HttpResponseMessage resp, string expected) string contentDigest; try { - contentDigest = DigestUtility.Parse(digestStr.FirstOrDefault()); + contentDigest = DigestUtility.ParseDigest(digestStr); } catch (Exception) { - throw new Exception( - $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: {DockerContentDigestHeader}: {digestStr}" - ); + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); } - if (contentDigest != expected) { - throw new Exception( -$"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in {DockerContentDigestHeader}: received {contentDigest}, while expecting {expected}" - ); + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); } - return; } @@ -431,24 +401,14 @@ public IManifestStore Manifests() /// public RemoteReference ParseReference(string reference) { + RemoteReference remoteReference; try { - var refObj = new RemoteReference().ParseReference(reference); - if (refObj.Registry != RemoteReference.Registry || refObj.Repository != RemoteReference.Repository) - { - throw new InvalidReferenceException( - $"mismatch between received {JsonSerializer.Serialize(refObj)} and expected {JsonSerializer.Serialize(RemoteReference)}"); - } - - if (string.IsNullOrEmpty(refObj.Reference)) - { - throw new InvalidReferenceException(); - } - return refObj; + remoteReference = RemoteReference.ParseReference(reference); } catch (Exception) { - var refObj = new RemoteReference + remoteReference = new RemoteReference { Registry = RemoteReference.Registry, Repository = RemoteReference.Repository, @@ -458,16 +418,28 @@ public RemoteReference ParseReference(string reference) if (reference.IndexOf("@") is var index && index != -1) { // `@` implies *digest*, so drop the *tag* (irrespective of what it is). - refObj.Reference = reference[(index + 1)..]; - refObj.ValidateReferenceAsDigest(); + remoteReference.Reference = reference[(index + 1)..]; + remoteReference.ValidateReferenceAsDigest(); } else { - refObj.ValidateReference(); + remoteReference.ValidateReference(); } - return refObj; + + } + + if (remoteReference.Registry != RemoteReference.Registry || remoteReference.Repository != RemoteReference.Repository) + { + throw new InvalidReferenceException( + $"mismatch between received {JsonSerializer.Serialize(remoteReference)} and expected {JsonSerializer.Serialize(RemoteReference)}"); } + if (string.IsNullOrEmpty(remoteReference.Reference)) + { + throw new InvalidReferenceException(); + } + return remoteReference; + } @@ -485,13 +457,13 @@ public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string { mediaType = "application/octet-stream"; } - var size = resp.Content.Headers.ContentLength!.Value; + var size = resp.Content.Headers.ContentLength.Value; if (size == -1) { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); } - RemoteReference.VerifyContentDigest(resp, refDigest); + VerifyContentDigest(resp, refDigest); return new Descriptor { @@ -522,9 +494,9 @@ public ManifestStore(Repository repository) /// public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var refObj = Repository.RemoteReference; - refObj.Reference = target.Digest; - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.RemoteReference; + remoteReference.Reference = target.Digest; + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", target.MediaType); var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); @@ -544,12 +516,13 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch response Content-Type {mediaType}: expect {target.MediaType}"); } - if (resp.Content.Headers!.ContentLength is var size && size != -1 && size != target.Size) + if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) { throw new Exception( $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } - RemoteReference.VerifyContentDigest(resp, target.Digest); + Repository.VerifyContentDigest(resp, target.Digest); + return await resp.Content.ReadAsStreamAsync(); } @@ -595,9 +568,9 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok /// private async Task PushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) { - var refObj = Repository.RemoteReference; - refObj.Reference = reference; - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.RemoteReference; + remoteReference.Reference = reference; + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Put, url); req.Content = new StreamContent(stream); req.Content.Headers.ContentLength = expected.Size; @@ -608,20 +581,20 @@ private async Task PushAsync(Descriptor expected, Stream stream, string referenc { throw await ErrorUtility.ParseErrorResponse(resp); } - RemoteReference.VerifyContentDigest(resp, expected.Digest); + Repository.VerifyContentDigest(resp, expected.Digest); } public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Head, url); req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); var res = await Repository.HttpClient.SendAsync(req, cancellationToken); return res.StatusCode switch { - HttpStatusCode.OK => await GenerateDescriptor(res, refObj, req.Method), + HttpStatusCode.OK => await GenerateDescriptor(res, remoteReference, req.Method), HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), _ => throw await ErrorUtility.ParseErrorResponse(res) }; @@ -650,7 +623,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote } // 2. Validate Size - if (!res.Content.Headers.ContentLength.HasValue) + if (!res.Content.Headers.ContentLength.HasValue || res.Content.Headers.ContentLength == -1) { throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: unknown response Content-Length"); } @@ -673,7 +646,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote { try { - RemoteReference.VerifyContentDigest(res, serverDigest); + Repository.VerifyContentDigest(res, serverDigest); } catch (Exception) { @@ -717,7 +690,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote { contentDigest = serverDigest; } - if (refDigest.Length > 0 && refDigest != contentDigest) + if (!string.IsNullOrEmpty(refDigest) && refDigest != contentDigest) { throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response; digest mismatch in {serverHeaderDigest}: received {contentDigest} when expecting {refDigest}"); } @@ -762,8 +735,8 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT /// public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); @@ -777,7 +750,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT } else { - desc = await GenerateDescriptor(resp, refObj, HttpMethod.Get); + desc = await GenerateDescriptor(resp, remoteReference, HttpMethod.Get); } return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: @@ -799,8 +772,8 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT public async Task PushReferenceAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); - await PushAsync(expected, content, refObj.Reference, cancellationToken); + var remoteReference = Repository.ParseReference(reference); + await PushAsync(expected, content, remoteReference.Reference, cancellationToken); } /// @@ -812,9 +785,9 @@ public async Task PushReferenceAsync(Descriptor expected, Stream content, string /// public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); + var remoteReference = Repository.ParseReference(reference); var rc = await FetchAsync(descriptor, cancellationToken); - await PushAsync(descriptor, rc, refObj.Reference, cancellationToken); + await PushAsync(descriptor, rc, remoteReference.Reference, cancellationToken); } } @@ -832,10 +805,10 @@ public BlobStore(Repository repository) public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - var refObj = Repository.RemoteReference; - DigestUtility.Parse(target.Digest); - refObj.Reference = target.Digest; - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.RemoteReference; + DigestUtility.ParseDigest(target.Digest); + remoteReference.Reference = target.Digest; + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { @@ -916,30 +889,26 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok // like registry.wabbit-networks.io:443/myrepo, blob push will fail since // the hostname of the Location header in the response is set to // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. - var uri = new Uri(location); + var uri = new UriBuilder(location); var locationHostname = uri.Host; var locationPort = uri.Port; // if location port 443 is missing, add it back if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) { - location = new Uri($"{locationHostname}:{reqPort}").ToString(); + location = new UriBuilder($"{locationHostname}:{reqPort}").ToString(); } url = location; var req = new HttpRequestMessage(HttpMethod.Put, url); req.Content = new StreamContent(content); - if (req.Content != null && req.Content.Headers.ContentLength is var size && size != expected.Size) - { - throw new Exception($"mismatch content length {size}: expect {expected.Size}"); - } req.Content.Headers.ContentLength = expected.Size; // the expected media type is ignored as in the API doc. req.Content.Headers.Add("Content-Type", "application/octet-stream"); // add digest key to query string with expected digest value - var query = ParseQueryString(new Uri(location).Query); + var query = ParseQueryString(new UriBuilder(location).Query); query.Add("digest", expected.Digest); req.RequestUri = new Uri($"{req.RequestUri}?digest={expected.Digest}"); @@ -966,15 +935,15 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok /// public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); - var refDigest = refObj.Digest(); - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.ParseReference(reference); + var refDigest = remoteReference.Digest(); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); return resp.StatusCode switch { HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), - HttpStatusCode.NotFound => throw new NotFoundException($"{refObj.Reference}: not found"), + HttpStatusCode.NotFound => throw new NotFoundException($"{remoteReference.Reference}: not found"), _ => throw await ErrorUtility.ParseErrorResponse(resp) }; } @@ -999,9 +968,9 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT /// public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) { - var refObj = Repository.ParseReference(reference); - var refDigest = refObj.Digest(); - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, refObj); + var remoteReference = Repository.ParseReference(reference); + var refDigest = remoteReference.Digest(); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs new file mode 100644 index 0000000..300b9fa --- /dev/null +++ b/Oras/Remote/ResponseTypes.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json.Serialization; + +namespace Oras.Remote +{ + internal static class ResponseTypes + { + internal class RepositoryList + { + [JsonPropertyName("repositories")] + public string[] Repositories { get; set; } + } + + internal class TagList + { + [JsonPropertyName("tags")] + public string[] Tags { get; set; } + } + } +} From 61c68fb7a00dc11900fa482522176ac17fcb0179 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 00:37:55 +0100 Subject: [PATCH 60/77] added using statements Signed-off-by: Samson Amaugo --- Oras.Tests/CopyFromRepositoryToMemory.cs | 26 +++++++ Oras.Tests/RemoteTest/RegistryTest.cs | 2 +- Oras.Tests/RemoteTest/RepositoryTest.cs | 2 +- Oras/Interfaces/Registry/IRegistry.cs | 7 +- Oras/Remote/Registry.cs | 13 ++-- Oras/Remote/RemoteReference.cs | 2 - Oras/Remote/Repository.cs | 92 +++++++++++------------- Oras/Remote/ResponseTypes.cs | 5 +- 8 files changed, 79 insertions(+), 70 deletions(-) create mode 100644 Oras.Tests/CopyFromRepositoryToMemory.cs diff --git a/Oras.Tests/CopyFromRepositoryToMemory.cs b/Oras.Tests/CopyFromRepositoryToMemory.cs new file mode 100644 index 0000000..46284bd --- /dev/null +++ b/Oras.Tests/CopyFromRepositoryToMemory.cs @@ -0,0 +1,26 @@ +using Moq; +using Moq.Protected; +using Xunit; + +namespace Oras.Tests +{ + public class CopyFromRepositoryToMemory + { + public static HttpClient CustomClient(Func func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).ReturnsAsync(func); + return new HttpClient(moqHandler.Object); + } + + [Fact] + public async Task Test_CopyFromRepositoryToMemory() + { + + } + } +} diff --git a/Oras.Tests/RemoteTest/RegistryTest.cs b/Oras.Tests/RemoteTest/RegistryTest.cs index 85870b5..5c0df99 100644 --- a/Oras.Tests/RemoteTest/RegistryTest.cs +++ b/Oras.Tests/RemoteTest/RegistryTest.cs @@ -1,9 +1,9 @@ using Moq; using Moq.Protected; +using Oras.Remote; using System.Net; using System.Text.Json; using System.Text.RegularExpressions; -using Oras.Remote; using Xunit; namespace Oras.Tests.RemoteTest diff --git a/Oras.Tests/RemoteTest/RepositoryTest.cs b/Oras.Tests/RemoteTest/RepositoryTest.cs index 9cda773..0296ddc 100644 --- a/Oras.Tests/RemoteTest/RepositoryTest.cs +++ b/Oras.Tests/RemoteTest/RepositoryTest.cs @@ -773,7 +773,7 @@ public async Task Repository_TagsAsync() break; } - var listOfTags = new ResponseTypes.TagList + var listOfTags = new ResponseTypes.TagList { Tags = tags.ToArray() }; diff --git a/Oras/Interfaces/Registry/IRegistry.cs b/Oras/Interfaces/Registry/IRegistry.cs index fc87ef7..df7a890 100644 --- a/Oras/Interfaces/Registry/IRegistry.cs +++ b/Oras/Interfaces/Registry/IRegistry.cs @@ -1,9 +1,6 @@ -using Oras.Remote; -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading.Tasks; +using System; using System.Threading; +using System.Threading.Tasks; namespace Oras.Interfaces.Registry { diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index bc9c256..be5e962 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -5,7 +5,6 @@ using System.Net.Http; using System.Text.Json; using System.Threading; -using Oras.Remote; using System.Threading.Tasks; using static System.Web.HttpUtility; @@ -20,7 +19,7 @@ public class Registry : IRegistry public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } - + public Registry(string name) { @@ -72,7 +71,7 @@ public async Task Repository(string name, CancellationToken cancell Registry = RemoteReference.Registry, Repository = name, }; - return new Repository(reference,HttpClient); + return new Repository(reference, HttpClient); } @@ -82,7 +81,7 @@ public async Task Repository(string name, CancellationToken cancell /// /// /// - public async Task Repository(string name,HttpClient httpClient, CancellationToken cancellationToken) + public async Task Repository(string name, HttpClient httpClient, CancellationToken cancellationToken) { var reference = new RemoteReference { @@ -130,8 +129,8 @@ public async Task Repositories(string last, Action fn, CancellationTok /// private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - - url = LinkUtils.ObtainUrl(url,PlainHTTP); + + url = LinkUtils.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -161,6 +160,6 @@ private async Task RepositoryPageAsync(string last, Action fn, return LinkUtils.ParseLink(response); } - + } } diff --git a/Oras/Remote/RemoteReference.cs b/Oras/Remote/RemoteReference.cs index d079cc6..57646cb 100644 --- a/Oras/Remote/RemoteReference.cs +++ b/Oras/Remote/RemoteReference.cs @@ -1,8 +1,6 @@ using Oras.Content; using Oras.Exceptions; using System; -using System.Linq; -using System.Net.Http; using System.Text.RegularExpressions; namespace Oras.Remote diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 608cda0..143184e 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -15,7 +15,7 @@ using static System.Web.HttpUtility; namespace Oras.Remote { - + /// /// Repository is an HTTP client to a remote repository /// @@ -60,7 +60,9 @@ public class Repository : IRepository /// public Repository(string reference) { - RemoteReference = RemoteReference.ParseReference(reference); ; + RemoteReference = RemoteReference.ParseReference(reference); + HttpClient = new HttpClient(); + HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); } /// @@ -73,27 +75,10 @@ public Repository(RemoteReference reference, HttpClient httpClient) { reference.ValidateRepository(); HttpClient = httpClient; + HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); RemoteReference = reference; } - /// - /// Client returns an HTTP client used to access the remote repository. - /// A default HTTP client is return if the client is not configured. - /// - /// - private HttpClient Client() - { - if (HttpClient is null) - { - var client = new HttpClient(); - client.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); - return client; - } - - return HttpClient; - } - - /// /// blobStore detects the blob store for the given descriptor. /// @@ -205,7 +190,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { await blobStore(target).DeleteAsync(target, cancellationToken); } - + /// /// TagsAsync returns a list of tags in a repository /// @@ -284,7 +269,7 @@ private async Task TagsPageAsync(string last, Action fn, strin } uriBuilder.Query = query.ToString(); - var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); + using var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); if (resp.StatusCode != HttpStatusCode.OK) { throw await ErrorUtility.ParseErrorResponse(resp); @@ -320,7 +305,7 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation url = URLUtiliity.BuildRepositoryBlobURL(PlainHTTP, remoteReference); } - var resp = await HttpClient.DeleteAsync(url, cancellationToken); + using var resp = await HttpClient.DeleteAsync(url, cancellationToken); switch (resp.StatusCode) { @@ -345,7 +330,7 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation /// internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) { - if (resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues) is var gotValues && !gotValues) return; + if (resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues) is var gotValues && !gotValues) return; var digestStr = digestValues.FirstOrDefault(); if (string.IsNullOrEmpty(digestStr)) { @@ -404,11 +389,11 @@ public RemoteReference ParseReference(string reference) RemoteReference remoteReference; try { - remoteReference = RemoteReference.ParseReference(reference); + remoteReference = RemoteReference.ParseReference(reference); } catch (Exception) { - remoteReference = new RemoteReference + remoteReference = new RemoteReference { Registry = RemoteReference.Registry, Repository = RemoteReference.Repository, @@ -425,9 +410,9 @@ public RemoteReference ParseReference(string reference) { remoteReference.ValidateReference(); } - + } - + if (remoteReference.Registry != RemoteReference.Registry || remoteReference.Repository != RemoteReference.Repository) { throw new InvalidReferenceException( @@ -499,7 +484,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", target.MediaType); - var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + using var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { @@ -522,8 +507,10 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } Repository.VerifyContentDigest(resp, target.Digest); - - return await resp.Content.ReadAsStreamAsync(); + var returnedStream = new MemoryStream(); + await resp.Content.CopyToAsync(returnedStream); + returnedStream.Seek(0, SeekOrigin.Begin); + return returnedStream; } /// @@ -576,7 +563,7 @@ private async Task PushAsync(Descriptor expected, Stream stream, string referenc req.Content.Headers.ContentLength = expected.Size; req.Content.Headers.Add("Content-Type", expected.MediaType); var client = Repository.HttpClient; - var resp = await client.SendAsync(req, cancellationToken); + using var resp = await client.SendAsync(req, cancellationToken); if (resp.StatusCode != HttpStatusCode.Created) { throw await ErrorUtility.ParseErrorResponse(resp); @@ -590,7 +577,7 @@ public async Task ResolveAsync(string reference, CancellationToken c var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Head, url); req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); - var res = await Repository.HttpClient.SendAsync(req, cancellationToken); + using var res = await Repository.HttpClient.SendAsync(req, cancellationToken); return res.StatusCode switch { @@ -638,7 +625,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote { } - + // 4. Validate Server Digest (if present) var serverHeaderDigest = res.Content.Headers.GetValues("Docker-Content-Digest"); var serverDigest = serverHeaderDigest.First(); @@ -646,7 +633,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote { try { - Repository.VerifyContentDigest(res, serverDigest); + Repository.VerifyContentDigest(res, serverDigest); } catch (Exception) { @@ -739,7 +726,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); - var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + using var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -752,7 +739,10 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = await GenerateDescriptor(resp, remoteReference, HttpMethod.Get); } - return (desc, await resp.Content.ReadAsStreamAsync()); + var returnedStream = new MemoryStream(); + await resp.Content.CopyToAsync(returnedStream); + returnedStream.Seek(0, SeekOrigin.Begin); + return (desc, returnedStream); case HttpStatusCode.NotFound: throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); default: @@ -809,7 +799,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel DigestUtility.ParseDigest(target.Digest); remoteReference.Reference = target.Digest; var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + using var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -818,8 +808,10 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } - - return await resp.Content.ReadAsStreamAsync(); + var returnedStream = new MemoryStream(); + await resp.Content.CopyToAsync(returnedStream); + returnedStream.Seek(0, SeekOrigin.Begin); + return returnedStream; case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); default: @@ -866,7 +858,7 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { var url = URLUtiliity.BuildRepositoryBlobUploadURL(Repository.PlainHTTP, Repository.RemoteReference); - var resp = await Repository.HttpClient.PostAsync(url, null, cancellationToken); + using var resp = await Repository.HttpClient.PostAsync(url, null, cancellationToken); var reqHostname = resp.RequestMessage.RequestUri.Host; var reqPort = resp.RequestMessage.RequestUri.Port; if (resp.StatusCode != HttpStatusCode.Accepted) @@ -918,10 +910,10 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok { req.Headers.Add("Authorization", auth.FirstOrDefault()); } - resp = await Repository.HttpClient.SendAsync(req, cancellationToken); - if (resp.StatusCode != HttpStatusCode.Created) + using var resp2 = await Repository.HttpClient.SendAsync(req, cancellationToken); + if (resp2.StatusCode != HttpStatusCode.Created) { - throw await ErrorUtility.ParseErrorResponse(resp); + throw await ErrorUtility.ParseErrorResponse(resp2); } return; @@ -939,7 +931,7 @@ public async Task ResolveAsync(string reference, CancellationToken c var refDigest = remoteReference.Digest(); var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); - var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); + using var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); return resp.StatusCode switch { HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), @@ -971,7 +963,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var remoteReference = Repository.ParseReference(reference); var refDigest = remoteReference.Digest(); var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + using var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -985,15 +977,15 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = Repository.GenerateBlobDescriptor(resp, refDigest); } - - return (desc, await resp.Content.ReadAsStreamAsync()); + var returnedStream = new MemoryStream(); + await resp.Content.CopyToAsync(returnedStream); + returnedStream.Seek(0, SeekOrigin.Begin); + return (desc, returnedStream); case HttpStatusCode.NotFound: throw new NotFoundException(); default: throw await ErrorUtility.ParseErrorResponse(resp); } } - } - } diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs index 300b9fa..f35794e 100644 --- a/Oras/Remote/ResponseTypes.cs +++ b/Oras/Remote/ResponseTypes.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Text.Json.Serialization; +using System.Text.Json.Serialization; namespace Oras.Remote { From 17c364217b58d5cadd9d5c03ef8d4a79c9b6e293 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 07:27:38 +0100 Subject: [PATCH 61/77] added extra test Signed-off-by: Samson Amaugo --- Oras.Tests/CopyFromRepositoryToMemory.cs | 122 +++++++++++++++++++++++ Oras/Content/DigestUtility.cs | 4 +- Oras/Models/Manifest.cs | 1 - 3 files changed, 124 insertions(+), 3 deletions(-) diff --git a/Oras.Tests/CopyFromRepositoryToMemory.cs b/Oras.Tests/CopyFromRepositoryToMemory.cs index 46284bd..546d6de 100644 --- a/Oras.Tests/CopyFromRepositoryToMemory.cs +++ b/Oras.Tests/CopyFromRepositoryToMemory.cs @@ -1,6 +1,14 @@ using Moq; using Moq.Protected; +using Oras.Constants; +using Oras.Interfaces; +using Oras.Memory; +using Oras.Models; +using Oras.Remote; +using System.Net; +using System.Text.Json; using Xunit; +using static Oras.Content.DigestUtility; namespace Oras.Tests { @@ -17,10 +25,124 @@ public static HttpClient CustomClient(Func + /// This test tries copying artifacts from the remote target to the memory target + /// + /// [Fact] public async Task Test_CopyFromRepositoryToMemory() { + var exampleManifest = @"hello world"u8.ToArray(); + var exampleManifestDescriptor = new Descriptor + { + MediaType = OCIMediaTypes.Descriptor, + Digest = CalculateSHA256DigestFromBytes(exampleManifest), + Size = exampleManifest.Length + }; + var exampleTag = "latest"; + var exampleUploadUUid = new Guid().ToString(); + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + var p = req.RequestUri.AbsolutePath; + var m = req.Method; + if (p.Contains("/blobs/uploads/") && m == HttpMethod.Post) + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Location = new Uri($"{p}/{exampleUploadUUid}"); + res.Content.Headers.ContentType.MediaType = OCIMediaTypes.ImageManifest; + return res; + } + if (p.Contains("/blobs/uploads/" + exampleUploadUUid) && m == HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (p.Contains("/manifests/latest") && m == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + if (p.Contains("/manifests/" + exampleManifestDescriptor.Digest) || p.Contains("/manifests/latest") && m == HttpMethod.Head) + { + if (m == HttpMethod.Get) + { + res.Content = new ByteArrayContent(exampleManifest); + res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); + res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); + res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + + + if (p.Contains("/blobs/") && (m == HttpMethod.Get || m == HttpMethod.Head)) + { + var arr = p.Split("/"); + var digest = arr[arr.Length - 1]; + Descriptor desc = null; + byte[] content = null; + + if (digest == exampleManifestDescriptor.Digest) + { + desc = exampleManifestDescriptor; + content = exampleManifest; + } + + res.Content = new ByteArrayContent(content); + res.Content.Headers.Add("Content-Type", desc.MediaType); + res.Content.Headers.Add("Docker-Content-Digest", digest); + res.Content.Headers.Add("Content-Length", content.Length.ToString()); + return res; + } + + if (p.Contains("/manifests/") && m == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return res; + }; + + var reg = new Registry("localhost:5000"); + + var src = await reg.Repository("source", CustomClient(func), CancellationToken.None); + var dst = new MemoryTarget(); + var tagName = "latest"; + var desc = await Copy.CopyAsync(src, tagName, dst, tagName, CancellationToken.None); + Console.WriteLine(desc.Digest); + } + + private async Task PushBlob(string mediaType, byte[] blob, ITarget target, CancellationToken cancellationToken = default) + { + var desc = new Descriptor + { + MediaType = mediaType, + Digest = CalculateSHA256DigestFromBytes(blob), + Size = blob.Length + }; + await target.PushAsync(desc, new MemoryStream(blob), cancellationToken); + return desc; } + + private byte[] GenerateManifestContent(Descriptor config, List layers) + { + var content = new Manifest + { + Config = config, + Layers = layers, + SchemaVersion = 2 + }; + return JsonSerializer.SerializeToUtf8Bytes(content); + } + } } diff --git a/Oras/Content/DigestUtility.cs b/Oras/Content/DigestUtility.cs index ec0407a..1ed1d02 100644 --- a/Oras/Content/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -36,8 +36,8 @@ internal static string CalculateSHA256DigestFromBytes(byte[] content) { using var sha256 = SHA256.Create(); var hash = sha256.ComputeHash(content); - var digest = $"sha256:{Convert.ToBase64String(hash)}"; - return digest; + var output = $"{nameof(SHA256)}:{BitConverter.ToString(hash).Replace("-", "")}"; + return output.ToLower(); } } } diff --git a/Oras/Models/Manifest.cs b/Oras/Models/Manifest.cs index 7714f75..e757214 100644 --- a/Oras/Models/Manifest.cs +++ b/Oras/Models/Manifest.cs @@ -20,7 +20,6 @@ public class Manifest [JsonPropertyName("annotations")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public Dictionary Annotations { get; set; } } } From f629f181c5348fdb8ac3463ea6518e47b117a5fd Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 07:28:37 +0100 Subject: [PATCH 62/77] added extra test Signed-off-by: Samson Amaugo --- Oras.Tests/CopyFromRepositoryToMemory.cs | 26 ------------------------ 1 file changed, 26 deletions(-) diff --git a/Oras.Tests/CopyFromRepositoryToMemory.cs b/Oras.Tests/CopyFromRepositoryToMemory.cs index 546d6de..5532a91 100644 --- a/Oras.Tests/CopyFromRepositoryToMemory.cs +++ b/Oras.Tests/CopyFromRepositoryToMemory.cs @@ -1,12 +1,10 @@ using Moq; using Moq.Protected; using Oras.Constants; -using Oras.Interfaces; using Oras.Memory; using Oras.Models; using Oras.Remote; using System.Net; -using System.Text.Json; using Xunit; using static Oras.Content.DigestUtility; @@ -120,29 +118,5 @@ public async Task Test_CopyFromRepositoryToMemory() var desc = await Copy.CopyAsync(src, tagName, dst, tagName, CancellationToken.None); Console.WriteLine(desc.Digest); } - - private async Task PushBlob(string mediaType, byte[] blob, ITarget target, CancellationToken cancellationToken = default) - { - var desc = new Descriptor - { - MediaType = mediaType, - Digest = CalculateSHA256DigestFromBytes(blob), - Size = blob.Length - }; - await target.PushAsync(desc, new MemoryStream(blob), cancellationToken); - return desc; - } - - private byte[] GenerateManifestContent(Descriptor config, List layers) - { - var content = new Manifest - { - Config = config, - Layers = layers, - SchemaVersion = 2 - }; - return JsonSerializer.SerializeToUtf8Bytes(content); - } - } } From bdce6be63df5d0e215e00dae0a1c95272adbece4 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 13:43:11 +0100 Subject: [PATCH 63/77] changed test location Signed-off-by: Samson Amaugo --- Oras.Tests/CopyFromRepositoryToMemory.cs | 122 ----------------------- Oras.Tests/RemoteTest/RepositoryTest.cs | 97 ++++++++++++++++++ 2 files changed, 97 insertions(+), 122 deletions(-) delete mode 100644 Oras.Tests/CopyFromRepositoryToMemory.cs diff --git a/Oras.Tests/CopyFromRepositoryToMemory.cs b/Oras.Tests/CopyFromRepositoryToMemory.cs deleted file mode 100644 index 5532a91..0000000 --- a/Oras.Tests/CopyFromRepositoryToMemory.cs +++ /dev/null @@ -1,122 +0,0 @@ -using Moq; -using Moq.Protected; -using Oras.Constants; -using Oras.Memory; -using Oras.Models; -using Oras.Remote; -using System.Net; -using Xunit; -using static Oras.Content.DigestUtility; - -namespace Oras.Tests -{ - public class CopyFromRepositoryToMemory - { - public static HttpClient CustomClient(Func func) - { - var moqHandler = new Mock(); - moqHandler.Protected().Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny() - ).ReturnsAsync(func); - return new HttpClient(moqHandler.Object); - } - - /// - /// This test tries copying artifacts from the remote target to the memory target - /// - /// - [Fact] - public async Task Test_CopyFromRepositoryToMemory() - { - var exampleManifest = @"hello world"u8.ToArray(); - - var exampleManifestDescriptor = new Descriptor - { - MediaType = OCIMediaTypes.Descriptor, - Digest = CalculateSHA256DigestFromBytes(exampleManifest), - Size = exampleManifest.Length - }; - var exampleTag = "latest"; - var exampleUploadUUid = new Guid().ToString(); - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - var p = req.RequestUri.AbsolutePath; - var m = req.Method; - if (p.Contains("/blobs/uploads/") && m == HttpMethod.Post) - { - res.StatusCode = HttpStatusCode.Accepted; - res.Headers.Location = new Uri($"{p}/{exampleUploadUUid}"); - res.Content.Headers.ContentType.MediaType = OCIMediaTypes.ImageManifest; - return res; - } - if (p.Contains("/blobs/uploads/" + exampleUploadUUid) && m == HttpMethod.Get) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - if (p.Contains("/manifests/latest") && m == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - if (p.Contains("/manifests/" + exampleManifestDescriptor.Digest) || p.Contains("/manifests/latest") && m == HttpMethod.Head) - { - if (m == HttpMethod.Get) - { - res.Content = new ByteArrayContent(exampleManifest); - res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); - res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); - res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - - - if (p.Contains("/blobs/") && (m == HttpMethod.Get || m == HttpMethod.Head)) - { - var arr = p.Split("/"); - var digest = arr[arr.Length - 1]; - Descriptor desc = null; - byte[] content = null; - - if (digest == exampleManifestDescriptor.Digest) - { - desc = exampleManifestDescriptor; - content = exampleManifest; - } - - res.Content = new ByteArrayContent(content); - res.Content.Headers.Add("Content-Type", desc.MediaType); - res.Content.Headers.Add("Docker-Content-Digest", digest); - res.Content.Headers.Add("Content-Length", content.Length.ToString()); - return res; - } - - if (p.Contains("/manifests/") && m == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return res; - }; - - var reg = new Registry("localhost:5000"); - - var src = await reg.Repository("source", CustomClient(func), CancellationToken.None); - var dst = new MemoryTarget(); - var tagName = "latest"; - var desc = await Copy.CopyAsync(src, tagName, dst, tagName, CancellationToken.None); - Console.WriteLine(desc.Digest); - } - } -} diff --git a/Oras.Tests/RemoteTest/RepositoryTest.cs b/Oras.Tests/RemoteTest/RepositoryTest.cs index 0296ddc..604f9b7 100644 --- a/Oras.Tests/RemoteTest/RepositoryTest.cs +++ b/Oras.Tests/RemoteTest/RepositoryTest.cs @@ -2,6 +2,7 @@ using Moq.Protected; using Oras.Constants; using Oras.Exceptions; +using Oras.Memory; using Oras.Models; using Oras.Remote; using System.Collections.Immutable; @@ -14,6 +15,7 @@ using System.Web; using Xunit; using static Oras.Content.Content; +using static Oras.Content.DigestUtility; namespace Oras.Tests.RemoteTest { @@ -1985,6 +1987,100 @@ public async Task ManifestStore_PushReferenceAsync() Assert.Equal(index, gotIndex); } + /// + /// This test tries copying artifacts from the remote target to the memory target + /// + /// + [Fact] + public async Task Test_CopyFromRepositoryToMemory() + { + var exampleManifest = @"hello world"u8.ToArray(); + + var exampleManifestDescriptor = new Descriptor + { + MediaType = OCIMediaTypes.Descriptor, + Digest = CalculateSHA256DigestFromBytes(exampleManifest), + Size = exampleManifest.Length + }; + var exampleTag = "latest"; + var exampleUploadUUid = new Guid().ToString(); + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + var p = req.RequestUri.AbsolutePath; + var m = req.Method; + if (p.Contains("/blobs/uploads/") && m == HttpMethod.Post) + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Location = new Uri($"{p}/{exampleUploadUUid}"); + res.Content.Headers.ContentType.MediaType = OCIMediaTypes.ImageManifest; + return res; + } + if (p.Contains("/blobs/uploads/" + exampleUploadUUid) && m == HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (p.Contains("/manifests/latest") && m == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + if (p.Contains("/manifests/" + exampleManifestDescriptor.Digest) || p.Contains("/manifests/latest") && m == HttpMethod.Head) + { + if (m == HttpMethod.Get) + { + res.Content = new ByteArrayContent(exampleManifest); + res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); + res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + res.Content.Headers.Add("Content-Type", OCIMediaTypes.Descriptor); + res.Content.Headers.Add("Docker-Content-Digest", exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + + + if (p.Contains("/blobs/") && (m == HttpMethod.Get || m == HttpMethod.Head)) + { + var arr = p.Split("/"); + var digest = arr[arr.Length - 1]; + Descriptor desc = null; + byte[] content = null; + + if (digest == exampleManifestDescriptor.Digest) + { + desc = exampleManifestDescriptor; + content = exampleManifest; + } + + res.Content = new ByteArrayContent(content); + res.Content.Headers.Add("Content-Type", desc.MediaType); + res.Content.Headers.Add("Docker-Content-Digest", digest); + res.Content.Headers.Add("Content-Length", content.Length.ToString()); + return res; + } + + if (p.Contains("/manifests/") && m == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return res; + }; + + var reg = new Registry("localhost:5000"); + + var src = await reg.Repository("source", CustomClient(func), CancellationToken.None); + var dst = new MemoryTarget(); + var tagName = "latest"; + var desc = await Copy.CopyAsync(src, tagName, dst, tagName, CancellationToken.None); + } public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { @@ -2045,5 +2141,6 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest } } + } } From f7a1f67949e6fbe7be0ee6d00879dd3fb8e56576 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 13:59:49 +0100 Subject: [PATCH 64/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Content/DigestUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Content/DigestUtility.cs b/Oras/Content/DigestUtility.cs index 1ed1d02..0cc73a0 100644 --- a/Oras/Content/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -21,7 +21,7 @@ internal static string ParseDigest(string digest) { if (!Regex.IsMatch(digest, digestRegexp)) { - throw new InvalidReferenceException($"Invalid digest: {digest}"); + throw new InvalidDigestException($"Invalid digest: {digest}"); } return digest; From 7a7dd666ba25f2ab074abd0ab6a064d0abc63f8f Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:10:19 +0100 Subject: [PATCH 65/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/{LinkUtils.cs => LinkUtility.cs} | 2 +- Oras/Remote/Registry.cs | 6 +++--- Oras/Remote/RemoteReference.cs | 4 ++-- Oras/Remote/Repository.cs | 6 +++--- Oras/Remote/ResponseTypes.cs | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) rename Oras/Remote/{LinkUtils.cs => LinkUtility.cs} (98%) diff --git a/Oras/Remote/LinkUtils.cs b/Oras/Remote/LinkUtility.cs similarity index 98% rename from Oras/Remote/LinkUtils.cs rename to Oras/Remote/LinkUtility.cs index 17de88b..cf2efd0 100644 --- a/Oras/Remote/LinkUtils.cs +++ b/Oras/Remote/LinkUtility.cs @@ -4,7 +4,7 @@ namespace Oras.Remote { - internal class LinkUtils + internal class LinkUtility { /// /// ParseLink returns the URL of the response's "Link" header, if present. diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index be5e962..e41bcf0 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -113,7 +113,7 @@ public async Task Repositories(string last, Action fn, CancellationTok last = ""; } } - catch (LinkUtils.NoLinkHeaderException) + catch (LinkUtility.NoLinkHeaderException) { return; } @@ -130,7 +130,7 @@ public async Task Repositories(string last, Action fn, CancellationTok private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - url = LinkUtils.ObtainUrl(url, PlainHTTP); + url = LinkUtility.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -157,7 +157,7 @@ private async Task RepositoryPageAsync(string last, Action fn, var data = await response.Content.ReadAsStringAsync(); var repositories = JsonSerializer.Deserialize(data); fn(repositories.Repositories); - return LinkUtils.ParseLink(response); + return LinkUtility.ParseLink(response); } diff --git a/Oras/Remote/RemoteReference.cs b/Oras/Remote/RemoteReference.cs index 57646cb..be2b32a 100644 --- a/Oras/Remote/RemoteReference.cs +++ b/Oras/Remote/RemoteReference.cs @@ -36,14 +36,14 @@ public class RemoteReference /// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53 /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - private static string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; + private const string repositoryRegexp = @"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$"; /// /// tagRegexp checks the tag name. /// The docker and OCI spec have the same regular expression. /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests /// - private static string tagRegexp = @"^[\w][\w.-]{0,127}$"; + private const string tagRegexp = @"^[\w][\w.-]{0,127}$"; public static RemoteReference ParseReference(string artifact) { diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 143184e..d58184b 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -234,7 +234,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken last = ""; } } - catch (LinkUtils.NoLinkHeaderException) + catch (LinkUtility.NoLinkHeaderException) { return; } @@ -251,7 +251,7 @@ public async Task TagsAsync(string last, Action fn, CancellationToken /// private async Task TagsPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - url = LinkUtils.ObtainUrl(url, PlainHTTP); + url = LinkUtility.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -278,7 +278,7 @@ private async Task TagsPageAsync(string last, Action fn, strin var data = await resp.Content.ReadAsStringAsync(); var tagList = JsonSerializer.Deserialize(data); fn(tagList.Tags); - return LinkUtils.ParseLink(resp); + return LinkUtility.ParseLink(resp); } /// diff --git a/Oras/Remote/ResponseTypes.cs b/Oras/Remote/ResponseTypes.cs index f35794e..0fdf200 100644 --- a/Oras/Remote/ResponseTypes.cs +++ b/Oras/Remote/ResponseTypes.cs @@ -4,13 +4,13 @@ namespace Oras.Remote { internal static class ResponseTypes { - internal class RepositoryList + internal struct RepositoryList { [JsonPropertyName("repositories")] public string[] Repositories { get; set; } } - internal class TagList + internal struct TagList { [JsonPropertyName("tags")] public string[] Tags { get; set; } From aa8af800d272e88c3fc2ad8fc53c163c2f3200f8 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:13:28 +0100 Subject: [PATCH 66/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Content/DigestUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Content/DigestUtility.cs b/Oras/Content/DigestUtility.cs index 0cc73a0..d5b4ea9 100644 --- a/Oras/Content/DigestUtility.cs +++ b/Oras/Content/DigestUtility.cs @@ -11,7 +11,7 @@ internal static class DigestUtility /// /// digestRegexp checks the digest. /// - private static string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; + private const string digestRegexp = @"[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+"; /// /// ParseDigest verifies the digest header and throws an exception if it is invalid. From 7d4f49df4b946407a71dbd03007d8a402c574a07 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:23:38 +0100 Subject: [PATCH 67/77] Update Oras/Remote/Repository.cs Co-authored-by: Lixia (Sylvia) Lei Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index d58184b..86f7821 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -19,7 +19,7 @@ namespace Oras.Remote /// /// Repository is an HTTP client to a remote repository /// - public class Repository : IRepository + public class Repository : IRepository, IRepositoryOption { /// /// HttpClient is the underlying HTTP client used to access the remote registry. From edf046ca1da89d63d03002de1edb1fdf5305937a Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:25:15 +0100 Subject: [PATCH 68/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Registry.cs | 2 +- Oras/Remote/Repository.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index e41bcf0..146687d 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -10,7 +10,7 @@ namespace Oras.Remote { - public class Registry : IRegistry + public class Registry : IRegistry, IRepositoryOption { public HttpClient HttpClient { get; set; } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 86f7821..cbd8162 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -80,11 +80,11 @@ public Repository(RemoteReference reference, HttpClient httpClient) } /// - /// blobStore detects the blob store for the given descriptor. + /// BlobStore detects the blob store for the given descriptor. /// /// /// - private IBlobStore blobStore(Descriptor desc) + private IBlobStore BlobStore(Descriptor desc) { if (ManifestUtility.IsManifest(ManifestMediaTypes, desc)) { @@ -104,7 +104,7 @@ private IBlobStore blobStore(Descriptor desc) /// public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) { - return await blobStore(target).FetchAsync(target, cancellationToken); + return await BlobStore(target).FetchAsync(target, cancellationToken); } /// @@ -115,7 +115,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel /// public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) { - return await blobStore(target).ExistsAsync(target, cancellationToken); + return await BlobStore(target).ExistsAsync(target, cancellationToken); } /// @@ -127,7 +127,7 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) { - await blobStore(expected).PushAsync(expected, content, cancellationToken); + await BlobStore(expected).PushAsync(expected, content, cancellationToken); } /// @@ -188,7 +188,7 @@ public async Task PushReferenceAsync(Descriptor descriptor, Stream content, stri /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) { - await blobStore(target).DeleteAsync(target, cancellationToken); + await BlobStore(target).DeleteAsync(target, cancellationToken); } /// From 74c2bee99dae40a261346848b9acf4550c51b747 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:26:57 +0100 Subject: [PATCH 69/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Registry.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 146687d..dbd46ab 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -19,8 +19,6 @@ public class Registry : IRegistry, IRepositoryOption public string[] ManifestMediaTypes { get; set; } public int TagListPageSize { get; set; } - - public Registry(string name) { var reference = new RemoteReference From a0818e5ff5e402eb69a519fc097a2dfb743a09be Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:56:10 +0100 Subject: [PATCH 70/77] Update Oras/Remote/Repository.cs Co-authored-by: Lixia (Sylvia) Lei Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index cbd8162..e003b5f 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -330,7 +330,7 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation /// internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) { - if (resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues) is var gotValues && !gotValues) return; + if (!resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues)) var digestStr = digestValues.FirstOrDefault(); if (string.IsNullOrEmpty(digestStr)) { From 27e1e7d69afb837fa6521da538d91e92bce533cf Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 14:57:23 +0100 Subject: [PATCH 71/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index e003b5f..4a58846 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -330,7 +330,7 @@ internal async Task DeleteAsync(Descriptor target, bool isManifest, Cancellation /// internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) { - if (!resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues)) + if (!resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues)) return; var digestStr = digestValues.FirstOrDefault(); if (string.IsNullOrEmpty(digestStr)) { From 6a54ac49689890c4ab2dea19a338168863d93e8a Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 15:12:37 +0100 Subject: [PATCH 72/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 4a58846..5e9f215 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -413,11 +413,6 @@ public RemoteReference ParseReference(string reference) } - if (remoteReference.Registry != RemoteReference.Registry || remoteReference.Repository != RemoteReference.Repository) - { - throw new InvalidReferenceException( - $"mismatch between received {JsonSerializer.Serialize(remoteReference)} and expected {JsonSerializer.Serialize(RemoteReference)}"); - } if (string.IsNullOrEmpty(remoteReference.Reference)) { From 2676ecb14f0b5fc5307cd653b1e23018bb816349 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 15:24:02 +0100 Subject: [PATCH 73/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 5e9f215..f28f601 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -413,7 +413,6 @@ public RemoteReference ParseReference(string reference) } - if (string.IsNullOrEmpty(remoteReference.Reference)) { throw new InvalidReferenceException(); @@ -691,7 +690,7 @@ public async Task GenerateDescriptor(HttpResponseMessage res, Remote /// taking care not to destroy it in the process /// /// - private async Task CalculateDigestFromResponse(HttpResponseMessage res) + static async Task CalculateDigestFromResponse(HttpResponseMessage res) { var bytes = await res.Content.ReadAsByteArrayAsync(); return DigestUtility.CalculateSHA256DigestFromBytes(bytes); From ee0af2ca62e9432a51641d3f8e053d15b1387eae Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 15:45:39 +0100 Subject: [PATCH 74/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index f28f601..9aec625 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -259,8 +259,6 @@ private async Task TagsPageAsync(string last, Action fn, strin if (TagListPageSize > 0) { query["n"] = TagListPageSize.ToString(); - - } if (last != "") { @@ -478,7 +476,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", target.MediaType); - using var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { @@ -501,10 +499,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } Repository.VerifyContentDigest(resp, target.Digest); - var returnedStream = new MemoryStream(); - await resp.Content.CopyToAsync(returnedStream); - returnedStream.Seek(0, SeekOrigin.Begin); - return returnedStream; + return await resp.Content.ReadAsStreamAsync(); } /// @@ -720,7 +715,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); - using var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -733,10 +728,8 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = await GenerateDescriptor(resp, remoteReference, HttpMethod.Get); } - var returnedStream = new MemoryStream(); - await resp.Content.CopyToAsync(returnedStream); - returnedStream.Seek(0, SeekOrigin.Begin); - return (desc, returnedStream); + + return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); default: @@ -793,7 +786,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel DigestUtility.ParseDigest(target.Digest); remoteReference.Reference = target.Digest; var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - using var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -802,10 +795,7 @@ public async Task FetchAsync(Descriptor target, CancellationToken cancel { throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); } - var returnedStream = new MemoryStream(); - await resp.Content.CopyToAsync(returnedStream); - returnedStream.Seek(0, SeekOrigin.Begin); - return returnedStream; + return await resp.Content.ReadAsStreamAsync(); case HttpStatusCode.NotFound: throw new NotFoundException($"{target.Digest}: not found"); default: @@ -894,9 +884,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok req.Content.Headers.Add("Content-Type", "application/octet-stream"); // add digest key to query string with expected digest value - var query = ParseQueryString(new UriBuilder(location).Query); - query.Add("digest", expected.Digest); - req.RequestUri = new Uri($"{req.RequestUri}?digest={expected.Digest}"); + req.RequestUri = new UriBuilder($"{req.RequestUri}?digest={expected.Digest}").Uri; //reuse credential from previous POST request resp.Headers.TryGetValues("Authorization", out var auth); @@ -957,7 +945,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT var remoteReference = Repository.ParseReference(reference); var refDigest = remoteReference.Digest(); var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - using var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); switch (resp.StatusCode) { case HttpStatusCode.OK: @@ -971,10 +959,8 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = Repository.GenerateBlobDescriptor(resp, refDigest); } - var returnedStream = new MemoryStream(); - await resp.Content.CopyToAsync(returnedStream); - returnedStream.Seek(0, SeekOrigin.Begin); - return (desc, returnedStream); + + return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException(); default: From 31595ef9f37745c6373c7f07f8a886c1160c3824 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 16:01:57 +0100 Subject: [PATCH 75/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/LinkUtility.cs | 25 ------------------------- Oras/Remote/Registry.cs | 4 ---- Oras/Remote/Repository.cs | 5 ++--- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/Oras/Remote/LinkUtility.cs b/Oras/Remote/LinkUtility.cs index cf2efd0..5482c53 100644 --- a/Oras/Remote/LinkUtility.cs +++ b/Oras/Remote/LinkUtility.cs @@ -49,31 +49,6 @@ internal static string ParseLink(HttpResponseMessage resp) return resolvedUri.AbsoluteUri; } - /// - /// ObtainUrl returns the link with a scheme and authority. - /// - /// - /// - /// - internal static string ObtainUrl(string url, bool plainHttp) - { - if (plainHttp) - { - if (!url.Contains("http")) - { - url = "http://" + url; - } - } - else - { - if (!url.Contains("https")) - { - url = "https://" + url; - } - } - - return url; - } /// /// NoLinkHeaderException is thrown when a link header is missing. diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index dbd46ab..15c9a80 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -127,8 +127,6 @@ public async Task Repositories(string last, Action fn, CancellationTok /// private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - - url = LinkUtility.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -157,7 +155,5 @@ private async Task RepositoryPageAsync(string last, Action fn, fn(repositories.Repositories); return LinkUtility.ParseLink(response); } - - } } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 9aec625..89f68d5 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -251,7 +251,6 @@ public async Task TagsAsync(string last, Action fn, CancellationToken /// private async Task TagsPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) { - url = LinkUtility.ObtainUrl(url, PlainHTTP); var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); if (TagListPageSize > 0 || last != "") @@ -728,7 +727,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = await GenerateDescriptor(resp, remoteReference, HttpMethod.Get); } - + return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); @@ -959,7 +958,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT { desc = Repository.GenerateBlobDescriptor(resp, refDigest); } - + return (desc, await resp.Content.ReadAsStreamAsync()); case HttpStatusCode.NotFound: throw new NotFoundException(); From 3cf1bcf0ecb092040f7d32ffd311d7bae803ed29 Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 16:27:46 +0100 Subject: [PATCH 76/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras/Remote/Repository.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 89f68d5..1aa3545 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -384,12 +384,14 @@ public IManifestStore Manifests() public RemoteReference ParseReference(string reference) { RemoteReference remoteReference; + var hasError = false; try { remoteReference = RemoteReference.ParseReference(reference); } catch (Exception) { + hasError = true; remoteReference = new RemoteReference { Registry = RemoteReference.Registry, @@ -410,6 +412,15 @@ public RemoteReference ParseReference(string reference) } + if (!hasError) + { + if (remoteReference.Registry != RemoteReference.Registry || + remoteReference.Repository != RemoteReference.Repository) + { + throw new InvalidReferenceException( + $"mismatch between received {JsonSerializer.Serialize(remoteReference)} and expected {JsonSerializer.Serialize(RemoteReference)}"); + } + } if (string.IsNullOrEmpty(remoteReference.Reference)) { throw new InvalidReferenceException(); From 3dc8dda857b54a08ca0318cc42a54719db04c76b Mon Sep 17 00:00:00 2001 From: Samson Amaugo Date: Sat, 27 May 2023 17:06:07 +0100 Subject: [PATCH 77/77] resolved an issue Signed-off-by: Samson Amaugo --- Oras.Tests/RemoteTest/RepositoryTest.cs | 5 ++-- Oras/Remote/Registry.cs | 34 ++++++++++--------------- Oras/Remote/Repository.cs | 21 ++++++++++++--- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/Oras.Tests/RemoteTest/RepositoryTest.cs b/Oras.Tests/RemoteTest/RepositoryTest.cs index 604f9b7..6cbead0 100644 --- a/Oras.Tests/RemoteTest/RepositoryTest.cs +++ b/Oras.Tests/RemoteTest/RepositoryTest.cs @@ -2075,8 +2075,9 @@ public async Task Test_CopyFromRepositoryToMemory() }; var reg = new Registry("localhost:5000"); - - var src = await reg.Repository("source", CustomClient(func), CancellationToken.None); + reg.HttpClient = CustomClient(func); + var src = await reg.Repository("source", CancellationToken.None); + var dst = new MemoryTarget(); var tagName = "latest"; var desc = await Copy.CopyAsync(src, tagName, dst, tagName, CancellationToken.None); diff --git a/Oras/Remote/Registry.cs b/Oras/Remote/Registry.cs index 15c9a80..4581b9c 100644 --- a/Oras/Remote/Registry.cs +++ b/Oras/Remote/Registry.cs @@ -31,6 +31,17 @@ public Registry(string name) HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); } + public Registry(string name, HttpClient httpClient) + { + var reference = new RemoteReference + { + Registry = name, + }; + reference.ValidateRegistry(); + RemoteReference = reference; + HttpClient = httpClient; + } + /// /// PingAsync checks whether or not the registry implement Docker Registry API V2 or /// OCI Distribution Specification. @@ -69,27 +80,8 @@ public async Task Repository(string name, CancellationToken cancell Registry = RemoteReference.Registry, Repository = name, }; - return new Repository(reference, HttpClient); - } - - - /// - /// Repository returns a repository object for the given repository name. - /// - /// - /// - /// - public async Task Repository(string name, HttpClient httpClient, CancellationToken cancellationToken) - { - var reference = new RemoteReference - { - Registry = RemoteReference.Registry, - Repository = name, - }; - HttpClient = httpClient; - HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); - - return new Repository(reference, HttpClient); + + return new Repository(reference, this); } diff --git a/Oras/Remote/Repository.cs b/Oras/Remote/Repository.cs index 1aa3545..ebcaf41 100644 --- a/Oras/Remote/Repository.cs +++ b/Oras/Remote/Repository.cs @@ -64,19 +64,32 @@ public Repository(string reference) HttpClient = new HttpClient(); HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); } + + /// + /// Creates a client to the remote repository using a reference and a HttpClient + /// + /// + /// + public Repository(string reference, HttpClient httpClient) + { + RemoteReference = RemoteReference.ParseReference(reference); + HttpClient = httpClient; + } /// /// This constructor customizes the HttpClient and sets the properties /// using values from the parameter. /// /// - /// - public Repository(RemoteReference reference, HttpClient httpClient) + /// + internal Repository(RemoteReference reference, IRepositoryOption option) { reference.ValidateRepository(); - HttpClient = httpClient; - HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); + HttpClient = option.HttpClient; RemoteReference = reference; + ManifestMediaTypes = option.ManifestMediaTypes; + PlainHTTP = option.PlainHTTP; + TagListPageSize = option.TagListPageSize; } ///