Prototype schema splitting into resources, data sources, provider and…

… core

flake.nix

@@ -187,7 +187,10 @@
 
           schemas = lib.mapAttrs
             (name: p: self.generateSchema.${system} (_: { ${name} = p; }))
-            terraformProviders;
+            terraformProviders
+          // {
+            terraform-core = self.generateSchema.${system} (_: { });
+          };
 
           lib = {
             mkDevShell =

tf-ncl/src/intermediate.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, convert::TryFrom};
 
 use serde::{Deserialize, Deserializer};
 
@@ -136,6 +136,46 @@ impl FieldDescriptor {
     }
 }
 
+#[derive(Debug)]
+pub struct SplitSchema {
+    pub resources: HashMap<String, Attribute>,
+    pub data_sources: HashMap<String, Attribute>,
+    pub provider_schema: HashMap<String, Attribute>,
+    pub core_schema: GoSchema,
+}
+
+#[derive(Debug)]
+pub enum SplittingError {
+    LeftoverComputedFields,
+    MissingBlock { field: &'static str },
+    ExpectedObject { field: &'static str },
+    ExpectedListOfObjects { field: String },
+    MissingProvider { provider: String },
+}
+
+impl Attribute {
+    pub fn into_object_content(self) -> Option<HashMap<String, Attribute>> {
+        match self.type_ {
+            Type::Object { open: _, content } => Some(content),
+            _ => None,
+        }
+    }
+
+    pub fn into_list_content(self) -> Option<HashMap<String, Attribute>> {
+        match self.type_ {
+            Type::List {
+                min: _,
+                max: _,
+                content,
+            } => match *content {
+                Type::Object { open: _, content } => Some(content),
+                _ => None,
+            },
+            _ => None,
+        }
+    }
+}
+
 impl GoSchema {
     pub fn push_down_computed_fields(self) -> Self {
         let Self {
@@ -150,4 +190,43 @@ impl GoSchema {
             schema,
         }
     }
+
+    pub fn split_for_provider(
+        mut self,
+        provider: impl AsRef<str>,
+    ) -> Result<SplitSchema, SplittingError> {
+        if !self.computed_fields.is_empty() {
+            return Err(SplittingError::LeftoverComputedFields);
+        }
+
+        Ok(SplitSchema {
+            resources: self
+                .schema
+                .remove("resource")
+                .ok_or(SplittingError::MissingBlock { field: "resource" })?
+                .into_object_content()
+                .ok_or(SplittingError::ExpectedObject { field: "resource" })?,
+            data_sources: self
+                .schema
+                .remove("data")
+                .ok_or(SplittingError::MissingBlock { field: "data" })?
+                .into_object_content()
+                .ok_or(SplittingError::ExpectedObject { field: "data" })?,
+            provider_schema: self
+                .schema
+                .remove("provider")
+                .ok_or(SplittingError::MissingBlock { field: "provider" })?
+                .into_object_content()
+                .ok_or(SplittingError::ExpectedObject { field: "provider" })?
+                .remove(provider.as_ref())
+                .ok_or(SplittingError::MissingProvider {
+                    provider: String::from(provider.as_ref()),
+                })?
+                .into_list_content()
+                .ok_or(SplittingError::ExpectedListOfObjects {
+                    field: format!("provider.{}", provider.as_ref()),
+                })?,
+            core_schema: self,
+        })
+    }
 }

tf-ncl/src/lib.rs

@@ -1,3 +1,4 @@
 pub mod intermediate;
 pub mod nickel;
 pub mod nickel_builder;
+pub mod terraform;

tf-ncl/src/main.rs

@@ -2,78 +2,82 @@ use clap::Parser;
 use core::fmt;
 use pretty::{BoxAllocator, BoxDoc, Pretty};
 use std::{
-    io::{self, stdout, Read},
-    path::PathBuf,
+    collections::HashMap,
+    fs,
+    io::{self, stdout, Write},
+    path::{Path, PathBuf},
 };
 use tf_ncl::{
     intermediate::{GoSchema, IntoWithProviders, Providers, WithProviders},
-    nickel::AsNickel,
+    nickel::{AsNickel, AsNickelTerm},
+    nickel_builder::{self as builder, Types},
+    terraform::TFSchema,
 };
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Args {
-    #[arg(value_name = "REQUIRED-PROVIDERS")]
-    providers: PathBuf,
-    #[arg(value_name = "TERRAFORM-SCHEMA")]
-    schema: Option<PathBuf>,
-}
-
-fn get_providers(opts: &Args) -> anyhow::Result<Providers> {
-    Ok(serde_json::from_reader(std::fs::File::open(
-        &opts.providers,
-    )?)?)
+    /// Terraform schema, generated by schema-merge
+    #[arg(value_name = "SCHEMA")]
+    schema: PathBuf,
+    /// Output directory for Nickel schema files
+    #[arg(value_name = "DIRECTORY")]
+    output_directory: PathBuf,
 }
 
 fn get_schema(opts: &Args) -> anyhow::Result<GoSchema> {
-    let schema_reader: Box<dyn Read> = if let Some(path) = &opts.schema {
-        Box::new(std::fs::File::open(path)?)
-    } else {
-        Box::new(std::io::stdin())
-    };
-
-    Ok(serde_json::from_reader(schema_reader)?)
+    Ok(serde_json::from_reader(std::fs::File::open(&opts.schema)?)?)
 }
 
 struct RenderableSchema<'a> {
-    schema: BoxDoc<'a>,
-    providers: BoxDoc<'a>,
+    schemas: HashMap<PathBuf, BoxDoc<'a>>,
 }
 
 impl<'a> RenderableSchema<'a> {
-    fn render(&self, f: &mut impl io::Write) -> anyhow::Result<()> {
+    fn render(&self, directory: &Path) -> anyhow::Result<()> {
         let tfncl_lib = include_str!("../../ncl/lib.ncl");
 
-        write!(
-            f,
-            "{{
-    Config = {{
-        config | Schema,
-        renderable_config = TfNcl.mkConfig config,
-        ..
-    }},
-    Schema = {schema},
-    TfNcl = {tfncl_lib} & {{
-        mkConfig = fun v => v |> TfNcl.resolve_provider_computed |> TfNcl.remove_empty_records,
-    }},
-    required_providers = {required_providers}
-}}",
-            schema = Display(&self.schema),
-            required_providers = Display(&self.providers),
-        )?;
-        Ok(())
-    }
-}
+        fs::create_dir_all(directory)?;
+        fs::write(directory.join("lib.ncl"), tfncl_lib)?;
 
-impl<'a> From<WithProviders<GoSchema>> for RenderableSchema<'a> {
-    fn from(s: WithProviders<GoSchema>) -> Self {
-        RenderableSchema {
-            schema: s.as_nickel().pretty(&BoxAllocator).into_doc(),
-            providers: s.providers.as_nickel().pretty(&BoxAllocator).into_doc(),
+        for (p, doc) in self.schemas.iter() {
+            let path = directory.join(p);
+            if let Some(parent) = path.parent() {
+                fs::create_dir_all(parent)?
+            }
+            write!(&mut fs::File::create(path)?, "{}", Display(doc))?;
         }
+
+        //         write!(
+        //             f,
+        //             "{{
+        //     Config = {{
+        //         config | Schema,
+        //         renderable_config = TfNcl.mkConfig config,
+        //         ..
+        //     }},
+        //     Schema = {schema},
+        //     TfNcl = {tfncl_lib} & {{
+        //         mkConfig = fun v => v |> TfNcl.resolve_provider_computed |> TfNcl.remove_empty_records,
+        //     }},
+        //     required_providers = {required_providers}
+        // }}",
+        //             schema = Display(&self.schema),
+        //             required_providers = Display(&self.providers),
+        //         )?;
+        Ok(())
     }
 }
 
+// impl<'a> From<WithProviders<GoSchema>> for RenderableSchema<'a> {
+//     fn from(s: WithProviders<GoSchema>) -> Self {
+//         RenderableSchema {
+//             schema: s.as_nickel().pretty(&BoxAllocator).into_doc(),
+//             providers: s.providers.as_nickel().pretty(&BoxAllocator).into_doc(),
+//         }
+//     }
+// }
+
 struct Display<'a, 'b>(&'b BoxDoc<'a>);
 
 impl<'a, 'b> fmt::Display for Display<'a, 'b> {
@@ -85,10 +89,21 @@ impl<'a, 'b> fmt::Display for Display<'a, 'b> {
 fn main() -> anyhow::Result<()> {
     let opts = Args::parse();
 
-    let providers = get_providers(&opts)?;
+    // let providers = get_providers(&opts)?;
     let go_schema = get_schema(&opts)?.push_down_computed_fields();
 
-    let doc: RenderableSchema = go_schema.with_providers(providers).into();
+    let split_schema = dbg!(go_schema.split_for_provider("github"));
+
+    Ok(())
+
+    // let doc: RenderableSchema = go_schema.with_providers(providers).into();
+
+    // let doc = RenderableSchema {
+    //     schemas: schema_docs
+    //         .into_iter()
+    //         .map(|(k, v)| (k, v.pretty(&BoxAllocator).into_doc()))
+    //         .collect(),
+    // };
 
-    doc.render(&mut stdout())
+    // doc.render(&opts.output_directory)
 }

tf-ncl/src/nickel.rs

@@ -1,26 +1,57 @@
+use std::collections::HashMap;
+use std::path::PathBuf;
 use std::rc::Rc;
 
 use crate::intermediate::{self, FieldDescriptor, GoSchema, Providers, WithProviders};
 use crate::nickel_builder::{self as builder, Types};
+use crate::terraform::{TFProviderSchema, TFSchema};
 use nickel_lang::term::array::{Array, ArrayAttrs};
 use nickel_lang::term::{MergePriority, RichTerm, Term};
 use nickel_lang::types::{DictTypeFlavour, TypeF};
 
 pub trait AsNickel {
-    fn as_nickel(&self) -> RichTerm;
+    fn as_nickel(&self, schemas: &mut HashMap<PathBuf, RichTerm>);
 }
 
-impl AsNickel for WithProviders<GoSchema> {
-    fn as_nickel(&self) -> RichTerm {
+impl AsNickel for TFProviderSchema {
+    fn as_nickel(&self, schemas: &mut HashMap<PathBuf, RichTerm>) {
+        let resources = self
+            .resource_schemas
+            .iter()
+            .map(|(resource, block_schema)| {
+                (
+                    PathBuf::new()
+                        .join("resource")
+                        .join(resource)
+                        .with_extension("ncl"),
+                    builder::Record::new().field("todo").no_value().build(),
+                )
+            });
+        let data = self.data_source_schemas.iter().map(|(data, block_schema)| {
+            (
+                PathBuf::new().join("data").join(data).with_extension("ncl"),
+                builder::Record::new().field("todo").no_value().build(),
+            )
+        });
+        schemas.extend(resources.chain(data))
+    }
+}
+
+pub trait AsNickelTerm {
+    fn as_nickel_term(&self) -> RichTerm;
+}
+
+impl AsNickelTerm for WithProviders<GoSchema> {
+    fn as_nickel_term(&self) -> RichTerm {
         as_nickel_record(&self.data.schema)
             .path(["terraform", "required_providers"])
-            .value(self.providers.as_nickel())
+            .value(self.providers.as_nickel_term())
             .build()
     }
 }
 
-impl AsNickel for Providers {
-    fn as_nickel(&self) -> RichTerm {
+impl AsNickelTerm for Providers {
+    fn as_nickel_term(&self) -> RichTerm {
         use builder::*;
         Record::from(self.0.iter().map(|(name, provider)| {
             Field::name(name).value(Record::from([
@@ -36,8 +67,8 @@ impl AsNickel for Providers {
     }
 }
 
-impl AsNickel for Vec<String> {
-    fn as_nickel(&self) -> RichTerm {
+impl AsNickelTerm for Vec<String> {
+    fn as_nickel_term(&self) -> RichTerm {
         Term::Array(
             Array::new(
                 self.iter()
@@ -52,12 +83,12 @@ impl AsNickel for Vec<String> {
     }
 }
 
-impl AsNickel for Vec<FieldDescriptor> {
-    fn as_nickel(&self) -> RichTerm {
+impl AsNickelTerm for Vec<FieldDescriptor> {
+    fn as_nickel_term(&self) -> RichTerm {
         Term::Array(
             Array::new(
                 self.iter()
-                    .map(|x| x.as_nickel())
+                    .map(|x| x.as_nickel_term())
                     .collect::<Vec<_>>()
                     .into_boxed_slice()
                     .into(),
@@ -68,8 +99,8 @@ impl AsNickel for Vec<FieldDescriptor> {
     }
 }
 
-impl AsNickel for FieldDescriptor {
-    fn as_nickel(&self) -> RichTerm {
+impl AsNickelTerm for FieldDescriptor {
+    fn as_nickel_term(&self) -> RichTerm {
         use builder::*;
 
         let priority = Term::Enum(if self.force {
@@ -211,8 +242,8 @@ impl AsNickelContracts for &intermediate::Type {
                     inner_contract,
                     Some(Types(TypeF::Flat(mk_app!(
                         Term::Var("TfNcl.ComputedFields".into()),
-                        prefix.as_nickel(),
-                        computed_fields.as_nickel()
+                        prefix.as_nickel_term(),
+                        computed_fields.as_nickel_term()
                     )))),
                 )
             }