From 6700924f2bff4081aaad8f64f99d9d5a0f665989 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 19:05:55 +0100 Subject: [PATCH 01/12] Add nix::fetchers::attrType --- .gitignore | 3 +++ Makefile | 1 + src/libfetchers/attrs.cc | 8 ++++++++ src/libfetchers/attrs.hh | 10 ++++++++++ src/libfetchers/tests/attrs.cc | 24 ++++++++++++++++++++++++ src/libfetchers/tests/local.mk | 29 +++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+) create mode 100644 src/libfetchers/tests/attrs.cc create mode 100644 src/libfetchers/tests/local.mk diff --git a/.gitignore b/.gitignore index d9f9d949b66..4897016bd2b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ perl/Makefile.config /src/libexpr/nix.tbl /tests/unit/libexpr/libnixexpr-tests +# /src/libfetchers/ +/src/libfetchers/tests/libnixfetchers-tests + # /src/libstore/ *.gen.* /tests/unit/libstore/libnixstore-tests diff --git a/Makefile b/Makefile index 1fdb6e89710..489fd672b8c 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ makefiles += \ tests/unit/libutil-support/local.mk \ tests/unit/libstore/local.mk \ tests/unit/libstore-support/local.mk \ + src/libfetchers/tests/local.mk \ tests/unit/libexpr/local.mk \ tests/unit/libexpr-support/local.mk endif diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index a565d19d4e7..3703aecd590 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -5,6 +5,14 @@ namespace nix::fetchers { +std::string attrType(const Attr &attr) { + return std::visit(overloaded { + [](const std::string &) { return "string"; }, + [](const uint64_t &) { return "int"; }, + [](const Explicit &) { return "bool"; }, + }, attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index b9a2c824ea7..9728be08b64 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -12,6 +12,9 @@ namespace nix::fetchers { +/** + * A primitive value that can be used in a fetcher attribute. + */ typedef std::variant> Attr; /** @@ -21,6 +24,13 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * A lowercase string designating the type of an `Attr`. + * + * Matches `builtins.typeOf` in Nix. + */ +std::string attrType(const Attr & attr); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libfetchers/tests/attrs.cc b/src/libfetchers/tests/attrs.cc new file mode 100644 index 00000000000..4b90ad6cdc0 --- /dev/null +++ b/src/libfetchers/tests/attrs.cc @@ -0,0 +1,24 @@ +#include "../attrs.hh" + +#include + +namespace nix::fetchers { + TEST(attrType, string0) { + ASSERT_EQ(attrType(""), "string"); + } + TEST(attrType, string1) { + ASSERT_EQ(attrType("hello"), "string"); + } + TEST(attrType, bool0) { + ASSERT_EQ(attrType(Explicit{false}), "bool"); + } + TEST(attrType, bool1) { + ASSERT_EQ(attrType(Explicit{true}), "bool"); + } + TEST(attrType, int0) { + ASSERT_EQ(attrType(0U), "int"); + } + TEST(attrType, int1) { + ASSERT_EQ(attrType(1U), "int"); + } +} \ No newline at end of file diff --git a/src/libfetchers/tests/local.mk b/src/libfetchers/tests/local.mk new file mode 100644 index 00000000000..ab21f31bd2a --- /dev/null +++ b/src/libfetchers/tests/local.mk @@ -0,0 +1,29 @@ +check: libfetchers-tests-exe_RUN + +programs += libfetchers-tests-exe + +libfetchers-tests-exe_NAME = libnixstore-tests + +libfetchers-tests-exe_DIR := $(d) + +libfetchers-tests-exe_INSTALL_DIR := + +libfetchers-tests-exe_LIBS = libfetchers-tests + +libfetchers-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libfetchers-tests + +libfetchers-tests_NAME = libnixfetchers-tests + +libfetchers-tests_DIR := $(d) + +libfetchers-tests_INSTALL_DIR := + +libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers-tests_CXXFLAGS += -I src/libfetchers -I src/libutil + +libfetchers-tests_LIBS = libutil-tests libfetchers libutil + +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) From 52a532813ba2da4af1470787e720d246c5778ceb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 19:41:44 +0100 Subject: [PATCH 02/12] Add nix::fetchers::parsers::String --- src/libfetchers/parser.cc | 19 +++++++++++++++ src/libfetchers/parser.hh | 41 ++++++++++++++++++++++++++++++++ src/libfetchers/schema.cc | 21 +++++++++++++++++ src/libfetchers/schema.hh | 42 +++++++++++++++++++++++++++++++++ src/libfetchers/tests/parser.cc | 28 ++++++++++++++++++++++ src/libfetchers/tests/schema.cc | 9 +++++++ 6 files changed, 160 insertions(+) create mode 100644 src/libfetchers/parser.cc create mode 100644 src/libfetchers/parser.hh create mode 100644 src/libfetchers/schema.cc create mode 100644 src/libfetchers/schema.hh create mode 100644 src/libfetchers/tests/parser.cc create mode 100644 src/libfetchers/tests/schema.cc diff --git a/src/libfetchers/parser.cc b/src/libfetchers/parser.cc new file mode 100644 index 00000000000..1a7c384b42b --- /dev/null +++ b/src/libfetchers/parser.cc @@ -0,0 +1,19 @@ +#include "parser.hh" +#include "schema.hh" + +namespace nix::fetchers { + + // parsers::String + + std::shared_ptr parsers::String::getSchema() { + return std::make_shared(Schema::Primitive::String); + } + + std::string parsers::String::parse (Attr in) { + std::string * r = std::get_if(&in); + if (r) + return *r; + else + throw Error("expected a string, but value is of type " + attrType(in)); + } +} \ No newline at end of file diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh new file mode 100644 index 00000000000..d6f54360afa --- /dev/null +++ b/src/libfetchers/parser.hh @@ -0,0 +1,41 @@ +#pragma once + +#include "attrs.hh" +#include + +namespace nix::fetchers { + + struct Schema; + + /** + A parser consists of + + - A function from a value of type In to a value of type Out + + - A nix::fetchers::Schema that describes what we want from the input of type In + */ + template + class Parser { + public: + /** The output type of the Parser. This type is particularly useful a template parameter is expected to be a Parser. */ + typedef Out_ Out; + + virtual std::shared_ptr getSchema() = 0; + virtual Out parse (In in) = 0; + }; + + namespace parsers { + + /** Accepts a string `Attr`. Rejects the other types. */ + class String : public Parser { + public: + std::shared_ptr getSchema() override; + std::string parse(Attr in) override; + }; + + std::string parse (Attr in) override; + }; + + } + +} \ No newline at end of file diff --git a/src/libfetchers/schema.cc b/src/libfetchers/schema.cc new file mode 100644 index 00000000000..98a2ba5f18f --- /dev/null +++ b/src/libfetchers/schema.cc @@ -0,0 +1,21 @@ +#include "schema.hh" + +namespace nix::fetchers { + + // bool Schema::Attrs::Attr::operator==(const Attr & other) const { + // return required == other.required && *type == *other.type; + // } + + // bool Schema::Attrs::operator==(const Attrs & other) const { + // return attrs == other.attrs; + // } + + // bool Schema::Union::operator==(const Union & other) const { + // return *a == *other.a && *b == *other.b; + // } + + bool Schema::operator==(const Schema & other) const { + return choice == other.choice; + } + +} diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh new file mode 100644 index 00000000000..f539eca4ec6 --- /dev/null +++ b/src/libfetchers/schema.hh @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +namespace nix::fetchers { + +/** + A schema is a data representation of the parsing logic that is applied to + nix::fetchers::Attrs. + + A Schema can be extracted from a nix::fetchers::Parser, and then be exported in JSON format. + + @todo: Add documentation fields + */ +struct Schema { + // struct Attrs { + // // struct Attr { + // // bool required; + // // std::shared_ptr type; + // // bool operator==(const Attr & other) const; + // // }; + // // std::map attrs; + // // bool operator==(const Attrs & other) const; + // }; + enum Primitive { + String, + // Int, + // Bool + }; + // struct Union { + // std::shared_ptr a; + // std::shared_ptr b; + // bool operator==(const Union & other) const; + // }; + + std::variant choice; + bool operator==(const Schema & other) const; +}; + +} diff --git a/src/libfetchers/tests/parser.cc b/src/libfetchers/tests/parser.cc new file mode 100644 index 00000000000..3d8c6d91c03 --- /dev/null +++ b/src/libfetchers/tests/parser.cc @@ -0,0 +1,28 @@ +#include "../parser.hh" +#include "../schema.hh" + +#include + +using namespace testing; + +namespace nix::fetchers { + using namespace parsers; + + TEST(String, example1) { + ASSERT_EQ(String{}.parse("hi"), "hi"); + } + TEST(String, intThrows) { + try { + String{}.parse(1U); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(e.what(), HasSubstr("expected a string, but value is of type int")); + } + } + TEST(String, schema) { + ASSERT_EQ( + *(String{}.getSchema()), + Schema { Schema::Primitive::String } + ); + } +} diff --git a/src/libfetchers/tests/schema.cc b/src/libfetchers/tests/schema.cc new file mode 100644 index 00000000000..db0ab22e8d1 --- /dev/null +++ b/src/libfetchers/tests/schema.cc @@ -0,0 +1,9 @@ +#include "../schema.hh" + +#include + +namespace nix::fetchers { + TEST(Schema_String, eq_String) { + ASSERT_EQ(Schema{Schema::String}, Schema{Schema::String}); + } +} From e3e8771f0a485c65fe025f1c7480d9441472db8d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 19:51:05 +0100 Subject: [PATCH 03/12] Add nix::fetchers::parsers::{Bool,Int} --- src/libfetchers/parser.cc | 28 ++++++++++++++++++++++++++++ src/libfetchers/parser.hh | 14 +++++++++++++- src/libfetchers/schema.hh | 4 ++-- src/libfetchers/tests/parser.cc | 30 ++++++++++++++++++++++++++++++ src/libfetchers/tests/schema.cc | 3 +++ 5 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/parser.cc b/src/libfetchers/parser.cc index 1a7c384b42b..73e12c36cf3 100644 --- a/src/libfetchers/parser.cc +++ b/src/libfetchers/parser.cc @@ -16,4 +16,32 @@ namespace nix::fetchers { else throw Error("expected a string, but value is of type " + attrType(in)); } + + // parsers::Int + + std::shared_ptr parsers::Int::getSchema() { + return std::make_shared(Schema::Primitive::Int); + } + + uint64_t parsers::Int::parse (Attr in) { + uint64_t * r = std::get_if(&in); + if (r) + return *r; + else + throw Error("expected an int, but value is of type " + attrType(in)); + } + + // parsers::Bool + + std::shared_ptr parsers::Bool::getSchema() { + return std::make_shared(Schema::Primitive::Bool); + } + + bool parsers::Bool::parse (Attr in) { + auto * r = std::get_if>(&in); + if (r) + return r->t; + else + throw Error("expected a bool, but value is of type " + attrType(in)); + } } \ No newline at end of file diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh index d6f54360afa..ed47d3fc84f 100644 --- a/src/libfetchers/parser.hh +++ b/src/libfetchers/parser.hh @@ -1,6 +1,7 @@ #pragma once #include "attrs.hh" +#include #include namespace nix::fetchers { @@ -33,7 +34,18 @@ namespace nix::fetchers { std::string parse(Attr in) override; }; - std::string parse (Attr in) override; + /** Accepts an int `Attr`. Rejects the other types. */ + class Int : public Parser { + public: + std::shared_ptr getSchema() override; + uint64_t parse(Attr in) override; + }; + + /** Accepts a bool `Attr`. Rejects the other types. */ + class Bool : public Parser { + public: + std::shared_ptr getSchema() override; + bool parse(Attr in) override; }; } diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh index f539eca4ec6..cabe4a08bd8 100644 --- a/src/libfetchers/schema.hh +++ b/src/libfetchers/schema.hh @@ -26,8 +26,8 @@ struct Schema { // }; enum Primitive { String, - // Int, - // Bool + Int, + Bool }; // struct Union { // std::shared_ptr a; diff --git a/src/libfetchers/tests/parser.cc b/src/libfetchers/tests/parser.cc index 3d8c6d91c03..3884c5ddb3e 100644 --- a/src/libfetchers/tests/parser.cc +++ b/src/libfetchers/tests/parser.cc @@ -25,4 +25,34 @@ namespace nix::fetchers { Schema { Schema::Primitive::String } ); } + + TEST(Int, example1) { + ASSERT_EQ(Int{}.parse(1U), 1U); + } + TEST(Int, stringThrows) { + try { + Int{}.parse("hi"); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(e.what(), HasSubstr("expected an int, but value is of type string")); + } + } + TEST(Int, schema) { + ASSERT_EQ( + *(Int{}.getSchema()), + Schema { Schema::Primitive::Int } + ); + } + + TEST(Bool, example1) { + ASSERT_EQ(Bool{}.parse(Explicit{true}), true); + } + TEST(Bool, stringThrows) { + try { + Bool{}.parse("hi"); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(e.what(), HasSubstr("expected a bool, but value is of type string")); + } + } } diff --git a/src/libfetchers/tests/schema.cc b/src/libfetchers/tests/schema.cc index db0ab22e8d1..3aa9d07e078 100644 --- a/src/libfetchers/tests/schema.cc +++ b/src/libfetchers/tests/schema.cc @@ -6,4 +6,7 @@ namespace nix::fetchers { TEST(Schema_String, eq_String) { ASSERT_EQ(Schema{Schema::String}, Schema{Schema::String}); } + TEST(Schema_String, neq_Int) { + ASSERT_NE(Schema{Schema::String}, Schema{Schema::Int}); + } } From 9cf092784a9668bea30ed0d1d4d15eef15e4a0cf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 20:27:19 +0100 Subject: [PATCH 04/12] Add nix::fetchers::Schema::Attrs --- src/libfetchers/schema.cc | 12 +++++------ src/libfetchers/schema.hh | 20 +++++++++--------- src/libfetchers/tests/schema.cc | 37 +++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/libfetchers/schema.cc b/src/libfetchers/schema.cc index 98a2ba5f18f..6ab20c864e4 100644 --- a/src/libfetchers/schema.cc +++ b/src/libfetchers/schema.cc @@ -2,13 +2,13 @@ namespace nix::fetchers { - // bool Schema::Attrs::Attr::operator==(const Attr & other) const { - // return required == other.required && *type == *other.type; - // } + bool Schema::Attrs::Attr::operator==(const Attr & other) const { + return required == other.required && *type == *other.type; + } - // bool Schema::Attrs::operator==(const Attrs & other) const { - // return attrs == other.attrs; - // } + bool Schema::Attrs::operator==(const Attrs & other) const { + return attrs == other.attrs; + } // bool Schema::Union::operator==(const Union & other) const { // return *a == *other.a && *b == *other.b; diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh index cabe4a08bd8..ac177a50e41 100644 --- a/src/libfetchers/schema.hh +++ b/src/libfetchers/schema.hh @@ -15,15 +15,15 @@ namespace nix::fetchers { @todo: Add documentation fields */ struct Schema { - // struct Attrs { - // // struct Attr { - // // bool required; - // // std::shared_ptr type; - // // bool operator==(const Attr & other) const; - // // }; - // // std::map attrs; - // // bool operator==(const Attrs & other) const; - // }; + struct Attrs { + struct Attr { + bool required; + std::shared_ptr type; + bool operator==(const Attr & other) const; + }; + std::map attrs; + bool operator==(const Attrs & other) const; + }; enum Primitive { String, Int, @@ -35,7 +35,7 @@ struct Schema { // bool operator==(const Union & other) const; // }; - std::variant choice; + std::variant choice; bool operator==(const Schema & other) const; }; diff --git a/src/libfetchers/tests/schema.cc b/src/libfetchers/tests/schema.cc index 3aa9d07e078..0c7f97a6dca 100644 --- a/src/libfetchers/tests/schema.cc +++ b/src/libfetchers/tests/schema.cc @@ -3,10 +3,47 @@ #include namespace nix::fetchers { + // Equality tests are boilerplaty but crucial for the validity of all tests, + // which use it. + TEST(Schema_String, eq_String) { ASSERT_EQ(Schema{Schema::String}, Schema{Schema::String}); } TEST(Schema_String, neq_Int) { ASSERT_NE(Schema{Schema::String}, Schema{Schema::Int}); } + TEST(Schema_String, neq_Attrs) { + ASSERT_NE(Schema{Schema::String}, Schema{Schema::Attrs{}}); + } + TEST(Schema_Attrs, eq_Attrs) { + ASSERT_EQ(Schema{Schema::Attrs{}}, Schema{Schema::Attrs{}}); + } + TEST(Schema_Attrs, neq_Attrs_attrType) { + Schema::Attrs a; + a.attrs.emplace("x", Schema::Attrs::Attr{true, std::make_shared(Schema::String)}); + Schema::Attrs b; + b.attrs.emplace("x", Schema::Attrs::Attr{true, std::make_shared(Schema::Int)}); + ASSERT_NE(Schema{a}, Schema{b}); + } + TEST(Schema_Attrs, neq_Attrs_attrName) { + Schema::Attrs a; + a.attrs.emplace("x", Schema::Attrs::Attr{true, std::make_shared(Schema::String)}); + Schema::Attrs b; + b.attrs.emplace("y", Schema::Attrs::Attr{true, std::make_shared(Schema::String)}); + ASSERT_NE(Schema{a}, Schema{b}); + } + TEST(Schema_Attrs, neq_Attrs_required) { + Schema::Attrs a; + a.attrs.emplace("x", Schema::Attrs::Attr{true, std::make_shared(Schema::String)}); + Schema::Attrs b; + b.attrs.emplace("x", Schema::Attrs::Attr{false, std::make_shared(Schema::String)}); + ASSERT_NE(Schema{a}, Schema{b}); + } + TEST(Schema_Attrs, neq_Attrs_missing) { + Schema::Attrs a; + a.attrs.emplace("x", Schema::Attrs::Attr{true, std::make_shared(Schema::String)}); + Schema::Attrs b; + ASSERT_NE(Schema{a}, Schema{b}); + } + } From 6f528f9a228157512a2d207dd05847adc45036ad Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 22:56:46 +0100 Subject: [PATCH 05/12] libutil: Add maybeGet to return optional from map lookup We already have some maybeGet*Attr functions - just not one for std::map itself yet. --- src/libutil/map.hh | 19 +++++++++++++++++++ tests/unit/libutil/tests.cc | 16 ++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 src/libutil/map.hh diff --git a/src/libutil/map.hh b/src/libutil/map.hh new file mode 100644 index 00000000000..b6b98277888 --- /dev/null +++ b/src/libutil/map.hh @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace nix { + + /** + * Return the value associated with the key, or `std::nullopt` if the key + * is not present. + */ + template + std::optional maybeGet(const std::map & m, const K & k) { + auto i = m.find(k); + if (i == m.end()) return std::nullopt; + return std::make_optional(i->second); + } + +} diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 568f03f702d..faf1226cdb3 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "types.hh" +#include "map.hh" #include "file-system.hh" #include "processes.hh" #include "terminal.hh" @@ -659,4 +660,19 @@ namespace nix { ASSERT_EQ(filterANSIEscapes("f๐ˆ๐ˆbรคr", true, 4), "f๐ˆ๐ˆb"); } + /* ---------------------------------------------------------------------------- + * maybeGet (map.hh) + * --------------------------------------------------------------------------*/ + + TEST(maybeGet, nested) { + std::map> m; + m["purgatory"] = std::nullopt; + m["hell"] = 666; + std::optional> awkward; + awkward.emplace(std::nullopt); + ASSERT_EQ(maybeGet(m, std::string{"nix"}), std::nullopt); + ASSERT_NE(maybeGet(m, std::string{"purgatory"}), std::nullopt); + ASSERT_EQ(maybeGet(m, std::string{"purgatory"}), awkward); + ASSERT_EQ(maybeGet(m, std::string{"hell"}), 666); + } } From cf3bc1987de63b17d34dcd9f2d3150b491cbcffd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 23:07:45 +0100 Subject: [PATCH 06/12] nix::fetchers::parsers: Make Attr fully qualified Name will be used in upcoming commit. --- src/libfetchers/parser.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/parser.cc b/src/libfetchers/parser.cc index 73e12c36cf3..38cf0224b99 100644 --- a/src/libfetchers/parser.cc +++ b/src/libfetchers/parser.cc @@ -9,7 +9,7 @@ namespace nix::fetchers { return std::make_shared(Schema::Primitive::String); } - std::string parsers::String::parse (Attr in) { + std::string parsers::String::parse (nix::fetchers::Attr in) { std::string * r = std::get_if(&in); if (r) return *r; @@ -23,7 +23,7 @@ namespace nix::fetchers { return std::make_shared(Schema::Primitive::Int); } - uint64_t parsers::Int::parse (Attr in) { + uint64_t parsers::Int::parse (nix::fetchers::Attr in) { uint64_t * r = std::get_if(&in); if (r) return *r; @@ -37,7 +37,7 @@ namespace nix::fetchers { return std::make_shared(Schema::Primitive::Bool); } - bool parsers::Bool::parse (Attr in) { + bool parsers::Bool::parse (nix::fetchers::Attr in) { auto * r = std::get_if>(&in); if (r) return r->t; From f571cb373a08c418e0e882ef6891514081e5b873 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 23:09:42 +0100 Subject: [PATCH 07/12] Add convenience initializer to fetchers::Schema::Attrs --- src/libfetchers/schema.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh index ac177a50e41..014c2d04f61 100644 --- a/src/libfetchers/schema.hh +++ b/src/libfetchers/schema.hh @@ -3,6 +3,7 @@ #include #include #include +#include namespace nix::fetchers { @@ -23,6 +24,10 @@ struct Schema { }; std::map attrs; bool operator==(const Attrs & other) const; + + Attrs() {}; + Attrs(std::map && attrs) + : attrs(attrs) {}; }; enum Primitive { String, From c25db0c93e08847a4804111dea4ce76cd2917acb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 23:09:12 +0100 Subject: [PATCH 08/12] Add nix::fetchers::parsers::Attrs etc --- src/libfetchers/parser.hh | 167 +++++++++++++++++++++++++++++++- src/libfetchers/tests/parser.cc | 144 +++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh index ed47d3fc84f..66c0ab76ec1 100644 --- a/src/libfetchers/parser.hh +++ b/src/libfetchers/parser.hh @@ -1,6 +1,9 @@ #pragma once #include "attrs.hh" +#include "error.hh" +#include "schema.hh" +#include "map.hh" #include #include @@ -48,6 +51,166 @@ namespace nix::fetchers { bool parse(Attr in) override; }; - } + template + class Attr : public Parser, Out>{ + public: + const std::string name; + Attr(std::string name) : name(name) {} + + virtual Out parse(std::optional in) override = 0; + + virtual bool isRequired() = 0; + + std::shared_ptr getSchema() override { + // Attributes aren't first class, so we won't be using this method. + // Perhaps use new superclass of Parser? Type parameter? + throw Error("not implemented"); + } + + virtual std::shared_ptr getAttrValueSchema() = 0; + + Schema::Attrs::Attr getAttrSchema() { + Schema::Attrs::Attr attrSchema; + attrSchema.type = getAttrValueSchema(); + attrSchema.required = isRequired(); + return attrSchema; + } + }; + + template + T parseAttr(const Attrs & attrs, Attr * parser) { + try { + return parser->parse(maybeGet(attrs, parser->name)); + } + catch (Error & e) { + e.addTrace(nullptr, "while checking fetcher attribute '%s'", parser->name); + throw e; + } + } + + template + class OptionalAttr : public Attr> { + Parser parser; + + public: + OptionalAttr(std::string name, Parser parser) + : Attr>(name), parser(parser) {} + + bool isRequired() override { return false; } + + std::optional parse(std::optional in) override { + // "map" + if (in) { + return parser.parse(*in); + } else { + return std::nullopt; + } + } + + std::shared_ptr getAttrValueSchema() override { + return parser.getSchema(); + } + }; + + template + class RequiredAttr : public Attr { + Parser parser; + + public: + RequiredAttr(std::string name, Parser parser) + : Attr(name), parser(parser) {} + + bool isRequired() override { return true; } + + typename Parser::Out parse(std::optional in) override { + if (in) { + return parser.parse(*in); + } else { + throw Error("required attribute '%s' not found", this->name); + } + } + + std::shared_ptr getAttrValueSchema() override { + return parser.getSchema(); + } + }; + + + /** + Perform a side effect for each item in a tuple. `f` must be callable for each item. + */ + template + void traverse_(F f, Tuple&& t) { + std::apply([&](auto&&... args) { (f(args), ...); }, t); + } + + /** Accepts an 'Attrs'. Composes 'Attr' (singular) parsers. */ + template + class Attrs + : public Parser< + nix::fetchers::Attrs, + std::invoke_result_t> { + Callable lambda; + std::tuple parsers; + std::shared_ptr schema; + Schema::Attrs * attrSchema; + + void checkUnknownAttrs(nix::fetchers::Attrs input) { + // Zip by key linearly. (Avoids the extra log term of find.) + auto iActual = input.begin(); + auto iExpected = attrSchema->attrs.begin(); + while (iExpected != attrSchema->attrs.end() && iActual != input.end()) { + if (iActual->first == iExpected->first) { + iExpected++; + iActual++; + } else if (iActual->first > iExpected->first) { + // (do nothing; .required is checked by the individual parser later) + iExpected++; + } else { + throw Error("unexpected attribute '%s'", iActual->first); + } + } + if (iActual != input.end()) { + throw Error("unexpected attribute '%s'", iActual->first); + } + } + + public: + Attrs(Callable lambda, AttrParsers *... parsers) + : lambda(lambda), parsers(parsers...) { + Schema::Attrs attrSchema; + + traverse_( + [this, &attrSchema](auto * parser) { + attrSchema.attrs[parser->name] = parser->getAttrSchema(); + }, + this->parsers + ); + + schema = std::make_shared(Schema{attrSchema}); + this->attrSchema = std::get_if(&schema->choice); + assert(this->attrSchema); + } + + std::invoke_result_t parse(nix::fetchers::Attrs input) override { + + checkUnknownAttrs(input); + + return std::apply( + [this, &input](auto *... parser) { + return lambda( + parseAttr(input, parser)... + ); + }, + parsers + ); + } + + std::shared_ptr getSchema() override { + return schema; + } + }; + + } // namespace parsers -} \ No newline at end of file +} // namespace nix::fetchers diff --git a/src/libfetchers/tests/parser.cc b/src/libfetchers/tests/parser.cc index 3884c5ddb3e..37de4004230 100644 --- a/src/libfetchers/tests/parser.cc +++ b/src/libfetchers/tests/parser.cc @@ -55,4 +55,148 @@ namespace nix::fetchers { ASSERT_THAT(e.what(), HasSubstr("expected a bool, but value is of type string")); } } + + auto attrsParser1 = parsers::Attrs( + [](auto a, auto b, auto c) { + return std::make_tuple(a, b, c); + }, + new RequiredAttr("a", String{}), + new OptionalAttr("b", Int{}), + new RequiredAttr("c", Bool{}) + ); + + TEST(Attrs, schema_attrsParser1) { + ASSERT_EQ( + *(attrsParser1.getSchema()), + Schema { + Schema::Attrs({ + { + std::string{"a"}, + Schema::Attrs::Attr { + true, + std::make_shared(Schema::Primitive::String) + } + }, + { + std::string{"b"}, + Schema::Attrs::Attr { + false, + std::make_shared(Schema::Primitive::Int) + } + }, + { + std::string{"c"}, + Schema::Attrs::Attr { + true, + std::make_shared(Schema::Primitive::Bool) + } + } + }) + } + ); + } + TEST(Attrs, parse_attrsParser1) { + ASSERT_EQ( + attrsParser1.parse( + nix::fetchers::Attrs{ + { "a", "hi" }, + { "b", 101U }, + { "c", Explicit{true} } + } + ), + std::make_tuple("hi", 101U, true) + ); + } + TEST(Attrs, parse_attrsParser1_missingOptional) { + ASSERT_EQ( + attrsParser1.parse( + nix::fetchers::Attrs{ + { "a", "hi" }, + { "c", Explicit{true} } + } + ), + std::make_tuple("hi", std::nullopt, true) + ); + } + TEST(Attrs, parse_attrsParser1_missingRequired) { + try { + attrsParser1.parse( + nix::fetchers::Attrs{ + { "b", 101U }, + { "c", Explicit{true} } + } + ); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("while checking fetcher attribute 'a'")); + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("required attribute 'a' not found")); + } + } + TEST(Attrs, parse_attrsParser1_wrongType) { + try { + attrsParser1.parse( + nix::fetchers::Attrs{ + { "a", "hi" }, + { "b", "hi" }, + { "c", Explicit{true} } + } + ); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("while checking fetcher attribute 'b'")); + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("expected an int, but value is of type string")); + } + } + TEST(Attrs, parse_attrsParser1_extra_before) { + try { + attrsParser1.parse( + nix::fetchers::Attrs{ + { "0", "hi" }, + { "a", "hi" }, + { "b", 101U }, + { "c", Explicit{true} } + } + ); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("unexpected attribute '0'")); + } + } + TEST(Attrs, parse_attrsParser1_extra_after) { + try { + attrsParser1.parse( + nix::fetchers::Attrs{ + { "a", "hi" }, + { "b", 101U }, + { "c", Explicit{true} }, + { "d", "hi" } + } + ); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("unexpected attribute 'd'")); + } + } + TEST(Attrs, parse_attrsParser1_extra_between) { + try { + attrsParser1.parse( + nix::fetchers::Attrs{ + { "a", "hi" }, + { "aa", "hi" }, + { "b", 101U }, + { "c", Explicit{true} } + } + ); + FAIL(); + } catch (Error & e) { + ASSERT_THAT(filterANSIEscapes(e.what(), true), + HasSubstr("unexpected attribute 'aa'")); + } + } } From 2f8f45676e23d5296c399a9fcafdcf52e3b901d5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 23:48:36 +0100 Subject: [PATCH 09/12] Add const to Parser::{parse,getSchema} --- src/libfetchers/parser.cc | 16 +++++++-------- src/libfetchers/parser.hh | 42 +++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/libfetchers/parser.cc b/src/libfetchers/parser.cc index 38cf0224b99..77677e16419 100644 --- a/src/libfetchers/parser.cc +++ b/src/libfetchers/parser.cc @@ -5,12 +5,12 @@ namespace nix::fetchers { // parsers::String - std::shared_ptr parsers::String::getSchema() { + std::shared_ptr parsers::String::getSchema() const { return std::make_shared(Schema::Primitive::String); } - std::string parsers::String::parse (nix::fetchers::Attr in) { - std::string * r = std::get_if(&in); + std::string parsers::String::parse (const nix::fetchers::Attr & in) const { + const std::string * r = std::get_if(&in); if (r) return *r; else @@ -19,12 +19,12 @@ namespace nix::fetchers { // parsers::Int - std::shared_ptr parsers::Int::getSchema() { + std::shared_ptr parsers::Int::getSchema() const { return std::make_shared(Schema::Primitive::Int); } - uint64_t parsers::Int::parse (nix::fetchers::Attr in) { - uint64_t * r = std::get_if(&in); + uint64_t parsers::Int::parse (const nix::fetchers::Attr & in) const { + const uint64_t * r = std::get_if(&in); if (r) return *r; else @@ -33,11 +33,11 @@ namespace nix::fetchers { // parsers::Bool - std::shared_ptr parsers::Bool::getSchema() { + std::shared_ptr parsers::Bool::getSchema() const { return std::make_shared(Schema::Primitive::Bool); } - bool parsers::Bool::parse (nix::fetchers::Attr in) { + bool parsers::Bool::parse (const nix::fetchers::Attr & in) const { auto * r = std::get_if>(&in); if (r) return r->t; diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh index 66c0ab76ec1..e700f8443ec 100644 --- a/src/libfetchers/parser.hh +++ b/src/libfetchers/parser.hh @@ -24,8 +24,8 @@ namespace nix::fetchers { /** The output type of the Parser. This type is particularly useful a template parameter is expected to be a Parser. */ typedef Out_ Out; - virtual std::shared_ptr getSchema() = 0; - virtual Out parse (In in) = 0; + virtual std::shared_ptr getSchema() const = 0; + virtual Out parse (const In & in) const = 0; }; namespace parsers { @@ -33,22 +33,22 @@ namespace nix::fetchers { /** Accepts a string `Attr`. Rejects the other types. */ class String : public Parser { public: - std::shared_ptr getSchema() override; - std::string parse(Attr in) override; + std::shared_ptr getSchema() const override; + std::string parse(const Attr & in) const override; }; /** Accepts an int `Attr`. Rejects the other types. */ class Int : public Parser { public: - std::shared_ptr getSchema() override; - uint64_t parse(Attr in) override; + std::shared_ptr getSchema() const override; + uint64_t parse(const Attr & in) const override; }; /** Accepts a bool `Attr`. Rejects the other types. */ class Bool : public Parser { public: - std::shared_ptr getSchema() override; - bool parse(Attr in) override; + std::shared_ptr getSchema() const override; + bool parse(const Attr & in) const override; }; template @@ -57,17 +57,17 @@ namespace nix::fetchers { const std::string name; Attr(std::string name) : name(name) {} - virtual Out parse(std::optional in) override = 0; + virtual Out parse(const std::optional & in) const override = 0; - virtual bool isRequired() = 0; + virtual bool isRequired() const = 0; - std::shared_ptr getSchema() override { + std::shared_ptr getSchema() const override { // Attributes aren't first class, so we won't be using this method. // Perhaps use new superclass of Parser? Type parameter? throw Error("not implemented"); } - virtual std::shared_ptr getAttrValueSchema() = 0; + virtual std::shared_ptr getAttrValueSchema() const = 0; Schema::Attrs::Attr getAttrSchema() { Schema::Attrs::Attr attrSchema; @@ -96,9 +96,9 @@ namespace nix::fetchers { OptionalAttr(std::string name, Parser parser) : Attr>(name), parser(parser) {} - bool isRequired() override { return false; } + bool isRequired() const override { return false; } - std::optional parse(std::optional in) override { + std::optional parse(const std::optional & in) const override { // "map" if (in) { return parser.parse(*in); @@ -107,7 +107,7 @@ namespace nix::fetchers { } } - std::shared_ptr getAttrValueSchema() override { + std::shared_ptr getAttrValueSchema() const override { return parser.getSchema(); } }; @@ -120,9 +120,9 @@ namespace nix::fetchers { RequiredAttr(std::string name, Parser parser) : Attr(name), parser(parser) {} - bool isRequired() override { return true; } + bool isRequired() const override { return true; } - typename Parser::Out parse(std::optional in) override { + typename Parser::Out parse(const std::optional & in) const override { if (in) { return parser.parse(*in); } else { @@ -130,7 +130,7 @@ namespace nix::fetchers { } } - std::shared_ptr getAttrValueSchema() override { + std::shared_ptr getAttrValueSchema() const override { return parser.getSchema(); } }; @@ -155,7 +155,7 @@ namespace nix::fetchers { std::shared_ptr schema; Schema::Attrs * attrSchema; - void checkUnknownAttrs(nix::fetchers::Attrs input) { + void checkUnknownAttrs(nix::fetchers::Attrs input) const { // Zip by key linearly. (Avoids the extra log term of find.) auto iActual = input.begin(); auto iExpected = attrSchema->attrs.begin(); @@ -192,7 +192,7 @@ namespace nix::fetchers { assert(this->attrSchema); } - std::invoke_result_t parse(nix::fetchers::Attrs input) override { + std::invoke_result_t parse(const nix::fetchers::Attrs & input) const override { checkUnknownAttrs(input); @@ -206,7 +206,7 @@ namespace nix::fetchers { ); } - std::shared_ptr getSchema() override { + std::shared_ptr getSchema() const override { return schema; } }; From eebdec5da7d59f5aae8a37f6c4bff520ce84e1a5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jan 2024 00:37:56 +0100 Subject: [PATCH 10/12] refact: Move src/libfetchers/tests to tests/unit/libfetchers --- Makefile | 2 +- {src => tests/unit}/libfetchers/tests/attrs.cc | 0 {src => tests/unit}/libfetchers/tests/local.mk | 0 {src => tests/unit}/libfetchers/tests/parser.cc | 0 {src => tests/unit}/libfetchers/tests/schema.cc | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename {src => tests/unit}/libfetchers/tests/attrs.cc (100%) rename {src => tests/unit}/libfetchers/tests/local.mk (100%) rename {src => tests/unit}/libfetchers/tests/parser.cc (100%) rename {src => tests/unit}/libfetchers/tests/schema.cc (100%) diff --git a/Makefile b/Makefile index 489fd672b8c..148f1260e6a 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ makefiles += \ tests/unit/libutil-support/local.mk \ tests/unit/libstore/local.mk \ tests/unit/libstore-support/local.mk \ - src/libfetchers/tests/local.mk \ + tests/unit/libfetchers/local.mk \ tests/unit/libexpr/local.mk \ tests/unit/libexpr-support/local.mk endif diff --git a/src/libfetchers/tests/attrs.cc b/tests/unit/libfetchers/tests/attrs.cc similarity index 100% rename from src/libfetchers/tests/attrs.cc rename to tests/unit/libfetchers/tests/attrs.cc diff --git a/src/libfetchers/tests/local.mk b/tests/unit/libfetchers/tests/local.mk similarity index 100% rename from src/libfetchers/tests/local.mk rename to tests/unit/libfetchers/tests/local.mk diff --git a/src/libfetchers/tests/parser.cc b/tests/unit/libfetchers/tests/parser.cc similarity index 100% rename from src/libfetchers/tests/parser.cc rename to tests/unit/libfetchers/tests/parser.cc diff --git a/src/libfetchers/tests/schema.cc b/tests/unit/libfetchers/tests/schema.cc similarity index 100% rename from src/libfetchers/tests/schema.cc rename to tests/unit/libfetchers/tests/schema.cc From 7fef87caecd0d32435bb561f89ee64d7cf53d1ed Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jan 2024 01:09:03 +0100 Subject: [PATCH 11/12] Fix compilation error --- src/libfetchers/parser.hh | 2 +- src/libfetchers/schema.hh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh index e700f8443ec..df5f20b3e5c 100644 --- a/src/libfetchers/parser.hh +++ b/src/libfetchers/parser.hh @@ -187,7 +187,7 @@ namespace nix::fetchers { this->parsers ); - schema = std::make_shared(Schema{attrSchema}); + schema = std::make_shared(attrSchema); this->attrSchema = std::get_if(&schema->choice); assert(this->attrSchema); } diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh index 014c2d04f61..92238c0f699 100644 --- a/src/libfetchers/schema.hh +++ b/src/libfetchers/schema.hh @@ -42,6 +42,9 @@ struct Schema { std::variant choice; bool operator==(const Schema & other) const; + + Schema(Primitive && p) : choice(p) {}; + Schema(Attrs && p) : choice(p) {}; }; } From f01441308df48bbf053428c45a8718314a3ab41e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jan 2024 20:08:54 +0100 Subject: [PATCH 12/12] WIP make it bidirectional and use for type = tarball --- src/libexpr/value.hh | 2 +- src/libfetchers/parser.cc | 12 +++ src/libfetchers/parser.hh | 146 +++++++++++++++++++++++++++++++++++-- src/libfetchers/schema.hh | 6 +- src/libfetchers/tarball.cc | 66 +++++++++++++++-- 5 files changed, 214 insertions(+), 18 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d9860e92194..08cff173092 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -7,7 +7,7 @@ #include "symbol-table.hh" #include "value/context.hh" -#include "input-accessor.hh" +#include "libfetchers/input-accessor.hh" #if HAVE_BOEHMGC #include diff --git a/src/libfetchers/parser.cc b/src/libfetchers/parser.cc index 77677e16419..97100e820db 100644 --- a/src/libfetchers/parser.cc +++ b/src/libfetchers/parser.cc @@ -17,6 +17,10 @@ namespace nix::fetchers { throw Error("expected a string, but value is of type " + attrType(in)); } + nix::fetchers::Attr parsers::String::unparse (const std::string & in) const { + return in; + } + // parsers::Int std::shared_ptr parsers::Int::getSchema() const { @@ -31,6 +35,10 @@ namespace nix::fetchers { throw Error("expected an int, but value is of type " + attrType(in)); } + nix::fetchers::Attr parsers::Int::unparse (const uint64_t & in) const { + return in; + } + // parsers::Bool std::shared_ptr parsers::Bool::getSchema() const { @@ -44,4 +52,8 @@ namespace nix::fetchers { else throw Error("expected a bool, but value is of type " + attrType(in)); } + + nix::fetchers::Attr parsers::Bool::unparse (const bool & in) const { + return Explicit{in}; + } } \ No newline at end of file diff --git a/src/libfetchers/parser.hh b/src/libfetchers/parser.hh index df5f20b3e5c..d5fd35171b9 100644 --- a/src/libfetchers/parser.hh +++ b/src/libfetchers/parser.hh @@ -2,10 +2,12 @@ #include "attrs.hh" #include "error.hh" +#include "libexpr/nixexpr.hh" #include "schema.hh" #include "map.hh" #include #include +#include namespace nix::fetchers { @@ -26,6 +28,11 @@ namespace nix::fetchers { virtual std::shared_ptr getSchema() const = 0; virtual Out parse (const In & in) const = 0; + virtual In unparse (const Out & out) const = 0; + virtual std::string show(const Out_ & out) const { + // FIXME + return ""; + }; }; namespace parsers { @@ -35,6 +42,8 @@ namespace nix::fetchers { public: std::shared_ptr getSchema() const override; std::string parse(const Attr & in) const override; + Attr unparse(const std::string & out) const override; + // std::string show(const std::string & out) const override; }; /** Accepts an int `Attr`. Rejects the other types. */ @@ -42,6 +51,8 @@ namespace nix::fetchers { public: std::shared_ptr getSchema() const override; uint64_t parse(const Attr & in) const override; + Attr unparse(const uint64_t & out) const override; + // std::string show(const uint64_t & out) const override; }; /** Accepts a bool `Attr`. Rejects the other types. */ @@ -49,8 +60,40 @@ namespace nix::fetchers { public: std::shared_ptr getSchema() const override; bool parse(const Attr & in) const override; + Attr unparse(const bool & out) const override; + // std::string show(const bool & out) const override; }; + // TODO + // template + // class Enum : public Parser { + // std::map values; + // std::map reverseValues; + + // public: + // Enum(std::map values) : values(values) {} + + // std::shared_ptr getSchema() const override { + // // FIXME + // throw Error("not implemented"); + // } + + // Out parse(const Attr & in) const override { + // auto it = values.find(in); + // if (it != values.end()) { + // return it->second; + // } else { + // throw Error("expected one of: %s", mapJoin(values, ", ", [](auto & pair) { + // return pair.first.toString(); + // })); + // } + // } + + // std::string show(const Out & out) const override { + // throw UnimplementedError("Enum.show"); + // } + // }; + template class Attr : public Parser, Out>{ public: @@ -59,6 +102,8 @@ namespace nix::fetchers { virtual Out parse(const std::optional & in) const override = 0; + // virtual std::optional unparse(const Out & out) const override = 0; + virtual bool isRequired() const = 0; std::shared_ptr getSchema() const override { @@ -69,10 +114,17 @@ namespace nix::fetchers { virtual std::shared_ptr getAttrValueSchema() const = 0; + virtual std::optional showDefaultValue() const { + return std::nullopt; + } + + // std::string show(const Out & out) const override; + Schema::Attrs::Attr getAttrSchema() { Schema::Attrs::Attr attrSchema; attrSchema.type = getAttrValueSchema(); attrSchema.required = isRequired(); + attrSchema.defaultValue = showDefaultValue(); return attrSchema; } }; @@ -88,13 +140,14 @@ namespace nix::fetchers { } } - template + template class OptionalAttr : public Attr> { Parser parser; + std::function(const From &)> restore; public: - OptionalAttr(std::string name, Parser parser) - : Attr>(name), parser(parser) {} + OptionalAttr(std::string name, Parser parser, std::function(const From &)> restore) + : Attr>(name), parser(parser), restore(restore) {} bool isRequired() const override { return false; } @@ -107,18 +160,32 @@ namespace nix::fetchers { } } + std::optional unparse(const std::optional & out) const override { + // "map" + if (out) { + return parser.unparse(*out); + } else { + return std::nullopt; + } + } + + std::optional unparseAttr(const From & out) const { + return unparse(restore(out)); + } + std::shared_ptr getAttrValueSchema() const override { return parser.getSchema(); } }; - template + template class RequiredAttr : public Attr { Parser parser; + std::function restore; public: - RequiredAttr(std::string name, Parser parser) - : Attr(name), parser(parser) {} + RequiredAttr(std::string name, Parser parser, std::function restore) + : Attr(name), parser(parser), restore(restore) {} bool isRequired() const override { return true; } @@ -130,11 +197,58 @@ namespace nix::fetchers { } } + std::optional unparse(const typename Parser::Out & out) const override { + return parser.unparse(out); + } + + std::optional unparseAttr(const From & out) const { + return unparse(restore(out)); + } + std::shared_ptr getAttrValueSchema() const override { return parser.getSchema(); } }; + template + class DefaultAttr : public Attr { + Parser parser; + typename Parser::Out defaultValue; + std::function restore; + + public: + DefaultAttr(std::string name, Parser parser, typename Parser::Out defaultValue, std::function restore) + : Attr(name), parser(parser), defaultValue(defaultValue), restore(restore) {} + + bool isRequired() const override { return false; } + + typename Parser::Out parse(const std::optional & in) const override { + if (in) { + return parser.parse(*in); + } else { + return defaultValue; + } + } + + std::optional unparse(const typename Parser::Out & out) const override { + // We might do this, but then the output is less useful. + // if (out == defaultValue) + // return std::nullopt; + return parser.unparse(out); + } + + std::optional unparseAttr(const From & out) const { + return unparse(restore(out)); + } + + std::optional showDefaultValue() const override { + return parser.show(defaultValue); + } + + std::shared_ptr getAttrValueSchema() const override { + return parser.getSchema(); + } + }; /** Perform a side effect for each item in a tuple. `f` must be callable for each item. @@ -181,13 +295,13 @@ namespace nix::fetchers { Schema::Attrs attrSchema; traverse_( - [this, &attrSchema](auto * parser) { + [&attrSchema](auto * parser) { attrSchema.attrs[parser->name] = parser->getAttrSchema(); }, this->parsers ); - schema = std::make_shared(attrSchema); + schema = std::make_shared(Schema(attrSchema)); this->attrSchema = std::get_if(&schema->choice); assert(this->attrSchema); } @@ -206,6 +320,22 @@ namespace nix::fetchers { ); } + nix::fetchers::Attrs unparse(const std::invoke_result_t & out) const override { + nix::fetchers::Attrs ret; + // for each of the parsers, unparse the output and add it to the attrs in ret + traverse_( + [&ret, &out](auto * parser) { + auto attr = parser->unparseAttr(out); + if (attr) { + ret[parser->name] = *attr; + } + }, + parsers + ); + + return ret; + } + std::shared_ptr getSchema() const override { return schema; } diff --git a/src/libfetchers/schema.hh b/src/libfetchers/schema.hh index 92238c0f699..4d6ac0548f1 100644 --- a/src/libfetchers/schema.hh +++ b/src/libfetchers/schema.hh @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -20,6 +21,7 @@ struct Schema { struct Attr { bool required; std::shared_ptr type; + std::optional defaultValue; bool operator==(const Attr & other) const; }; std::map attrs; @@ -43,8 +45,8 @@ struct Schema { std::variant choice; bool operator==(const Schema & other) const; - Schema(Primitive && p) : choice(p) {}; - Schema(Attrs && p) : choice(p) {}; + Schema(Primitive p) : choice(p) {}; + Schema(Attrs p) : choice(p) {}; }; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 3b7709440b2..8aa78fcdf4b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,6 +9,7 @@ #include "types.hh" #include "split.hh" #include "posix-source-accessor.hh" +#include "parser.hh" namespace nix::fetchers { @@ -183,6 +184,48 @@ DownloadTarballResult downloadTarball( }; } +struct TarballAttrs { + std::string url; + std::optional narHash; + std::optional name; + std::optional rev; + std::optional revCount; + std::optional lastModified; +}; + +static const auto tarballAttrParser = []{ + using namespace nix::fetchers::parsers; + return parsers::Attrs( + []( + std::string typ, + std::string url, + std::optional narHash, + // Had to make this optional to make the tests pass. fetchTree does not allow name? + std::optional name, + std::optional rev, + std::optional revCount, + std::optional lastModified + ) { + return TarballAttrs { + .url = url, + .narHash = narHash, + .name = name, + .rev = rev, + .revCount = revCount, + .lastModified = lastModified, + }; + }, + new DefaultAttr("type", String {} /* TODO check either missing or "tarball" */, "type", [](auto x) { return "tarball"; }), + new RequiredAttr("url", String {}, [](auto x) { return x.url; }), + new OptionalAttr("narHash", String {}, [](auto x) { return x.narHash; }), + // new DefaultAttr("name", String {}, "source", [](const TarballAttrs & x) { return x.name; }), + new OptionalAttr("name", String {}, [](auto x) { return x.name; }), + new OptionalAttr("rev", String {}, [](auto x) { return x.rev; }), + new OptionalAttr("revCount", Int {}, [](auto x) { return x.revCount; }), + new OptionalAttr("lastModified", Int {}, [](auto x) { return x.lastModified; }) + ); +}(); + // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { @@ -298,9 +341,11 @@ struct TarballInputScheme : CurlInputScheme std::pair fetch(ref store, const Input & _input) override { - Input input(_input); - auto url = getStrAttr(input.attrs, "url"); - auto result = downloadTarball(store, url, input.getName(), false); + std::cerr << "parsing input attrs" << std::endl; + std::cerr << _input.to_string() << std::endl; + TarballAttrs input = tarballAttrParser.parse(_input.attrs); + // Input input(_input); + auto result = downloadTarball(store, input.url, input.name ? *input.name : "source", false); if (result.immutableUrl) { auto immutableInput = Input::fromURL(*result.immutableUrl); @@ -308,13 +353,20 @@ struct TarballInputScheme : CurlInputScheme // here, e.g. git flakes. if (immutableInput.getType() != "tarball") throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported"); - input = immutableInput; + + return {result.storePath, immutableInput}; } - if (result.lastModified && !input.attrs.contains("lastModified")) - input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + if (result.lastModified && !input.lastModified) + input.lastModified = uint64_t(result.lastModified); + + Input input2; + input2.scheme = std::make_shared(); + input2.attrs = tarballAttrParser.unparse(input); + std::cerr << "unparsed" << std::endl; + std::cerr << input2.to_string() << std::endl; - return {result.storePath, std::move(input)}; + return {result.storePath, std::move(input2)}; } };