Enclave connections and enclave coordination in the C++ target

org.lflang/src/lib/cpp/lfutil.hh

@@ -27,6 +27,8 @@
 #include <reactor-cpp/reactor-cpp.hh>
 #include <reactor-cpp/logging.hh>
 
+#include <list>
+
 namespace lfutil {
 
 template<class T>

org.lflang/src/org/lflang/Target.java

@@ -527,7 +527,7 @@ public String getSingleLineCommentPrefix() {
      * program (and keepalive was not explicitly unset by the user).
      */
     public boolean setsKeepAliveOptionAutomatically() {
-        return this != Rust;
+        return this != Rust && this != CPP;
     }
 
     /**

org.lflang/src/org/lflang/generator/cpp/CppAssembleMethodGenerator.kt

@@ -26,8 +26,10 @@ package org.lflang.generator.cpp
 
 import org.lflang.*
 import org.lflang.generator.PrependOperator
+import org.lflang.generator.cpp.CppConnectionGenerator.Companion.isEnclaveConnection
 import org.lflang.generator.cpp.CppConnectionGenerator.Companion.name
 import org.lflang.generator.cpp.CppConnectionGenerator.Companion.requiresConnectionClass
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.isEnclave
 import org.lflang.generator.cpp.CppPortGenerator.Companion.dataType
 import org.lflang.lf.Action
 import org.lflang.lf.Connection
@@ -86,7 +88,7 @@ class CppAssembleMethodGenerator(private val reactor: Reactor) {
                     // is in a bank, but not a multiport
                     """
                         |for (auto& __lf_instance : ${container.name}) {
-                    ${" |  "..generateCode("__lf_instance->${port.name}")}
+                    ${" |  "..generateCode("__lf_instance->${if(container.isEnclave) "__lf_instance->" else ""}${port.name}")}
                         |}
                     """.trimMargin()
                 } else {
@@ -182,7 +184,7 @@ class CppAssembleMethodGenerator(private val reactor: Reactor) {
 
         // Generate code which adds all left hand ports and all right hand ports to a vector each. If we are handling multiports
         // within a bank, then we normally iterate over all banks in an outer loop and over all ports in an inner loop. However,
-        // if the connection is a cross connection, than we change the order on the right side and iterate over ports before banks.
+        // if the connection is an interleaved connection, than we change the order on the right side and iterate over ports before banks.
         return with(PrependOperator) {
             if (!c.requiresConnectionClass) {
                 """
@@ -193,7 +195,7 @@ class CppAssembleMethodGenerator(private val reactor: Reactor) {
                 ${" |"..c.rightPorts.joinWithLn { addAllPortsToVector(it, "__lf_right_ports_$idx") }}
                     |lfutil::bind_multiple_ports(__lf_left_ports_$idx, __lf_right_ports_$idx, ${c.isIterated});
                 """.trimMargin()
-            } else {
+            } else if (!c.isEnclaveConnection) {
                 """
                     |// connection $idx
                     |std::vector<$portType> __lf_left_ports_$idx;
@@ -211,6 +213,19 @@ class CppAssembleMethodGenerator(private val reactor: Reactor) {
                 ${" |"..c.rightPorts.joinWithLn { addAllPortsToVector(it, "__lf_right_ports_$idx") }}
                     |lfutil::bind_multiple_connections_with_ports(__lf_connection_pointers_$idx, __lf_right_ports_$idx, ${c.isIterated});
                 """.trimMargin()
+            } else {
+                """
+                    |// connection $idx
+                    |std::vector<$portType> __lf_left_ports_$idx;
+                ${" |"..c.leftPorts.joinWithLn { addAllPortsToVector(it, "__lf_left_ports_$idx") }}
+                    |std::vector<$portType> __lf_right_ports_$idx;
+                ${" |"..c.rightPorts.joinWithLn { addAllPortsToVector(it, "__lf_right_ports_$idx") }}
+                    |for(size_t __lf_idx{0}; __lf_idx < std::min(__lf_right_ports_$idx.size(), __lf_left_ports_$idx.size()); __lf_idx++) {
+                    |  ${c.name}.emplace_back("${c.name}" + std::to_string(__lf_idx), __lf_right_ports_$idx[__lf_idx]->environment()${if(c.delay != null) ", ${c.delay.toCppTime()}" else ""});
+                    |  ${c.name}.back().bind_upstream_port(__lf_left_ports_$idx[__lf_idx]);
+                    |  ${c.name}.back().bind_downstream_port(__lf_right_ports_$idx[__lf_idx]);
+                    |}
+                """.trimMargin()
             }
         }
     }

org.lflang/src/org/lflang/generator/cpp/CppConnectionGenerator.kt

@@ -2,6 +2,8 @@ package org.lflang.generator.cpp
 
 import org.lflang.*
 import org.lflang.generator.cpp.CppConnectionGenerator.Companion.cppType
+import org.lflang.generator.cpp.CppConnectionGenerator.Companion.isEnclaveConnection
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.isEnclave
 import org.lflang.generator.cpp.CppPortGenerator.Companion.dataType
 import org.lflang.lf.Connection
 import org.lflang.lf.Port
@@ -25,13 +27,38 @@ class CppConnectionGenerator(private val reactor: Reactor) {
             get() {
                 val leftPort = leftPorts.first()
                 return when {
-                    isPhysical    -> "reactor::PhysicalConnection<${leftPort.dataType}>"
-                    delay != null -> "reactor::DelayedConnection<${leftPort.dataType}>"
-                    else          -> throw IllegalArgumentException("Connection is neither physical nor delayed")
+                    isEnclaveConnection -> when {
+                        isPhysical    -> "reactor::PhysicalEnclaveConnection<${leftPort.dataType}>"
+                        delay != null -> "reactor::DelayedEnclaveConnection<${leftPort.dataType}>"
+                        else          -> "reactor::EnclaveConnection<${leftPort.dataType}>"
+                    }
+
+                    isPhysical          -> "reactor::PhysicalConnection<${leftPort.dataType}>"
+                    delay != null       -> "reactor::DelayedConnection<${leftPort.dataType}>"
+                    else                -> throw IllegalArgumentException("Unsupported connection type")
                 }
             }
 
-        val Connection.requiresConnectionClass: Boolean get() = isPhysical || delay != null
+        val Connection.isEnclaveConnection: Boolean
+            get() {
+                if (leftPorts.size == 1 && rightPorts.size == 1) {
+                    val leftIsEnclave = leftPorts[0].container?.isEnclave == true
+                    val rightIsEnclave = rightPorts[0].container?.isEnclave == true
+                    return when {
+                        leftIsEnclave && rightIsEnclave   -> true
+                        !leftIsEnclave && !rightIsEnclave -> false
+                        else                              -> TODO("Connections between enclaves and normal reactors are not supported")
+                    }
+                }
+                for (port in leftPorts + rightPorts) {
+                    if (port.container?.isEnclave == true) {
+                        TODO("Enclaves can only be used in simple connections")
+                    }
+                }
+                return false
+            }
+
+        val Connection.requiresConnectionClass: Boolean get() = isPhysical || delay != null || isEnclaveConnection;
     }
 
     fun generateDeclarations() =
@@ -42,18 +69,24 @@ class CppConnectionGenerator(private val reactor: Reactor) {
         reactor.connections.mapNotNull { generateConstructorInitializer(it) }.joinLn()
 
     private fun generateDecleration(connection: Connection): String? =
-        if (connection.requiresConnectionClass) {
-            if (connection.hasMultipleConnections) {
-                "std::vector<${connection.cppType}> ${connection.name};"
-            } else {
-                "${connection.cppType} ${connection.name};"
+        with(connection) {
+            if (requiresConnectionClass) {
+                when {
+                    hasMultipleConnections && isEnclaveConnection  -> "std::list<${connection.cppType}> ${connection.name};"
+                    hasMultipleConnections && !isEnclaveConnection -> "std::vector<${connection.cppType}> ${connection.name};"
+                    else                                           -> "${connection.cppType} ${connection.name};"
+                }
+            } else null
+        }
+
+    private fun generateConstructorInitializer(connection: Connection): String? = with(connection) {
+        if (requiresConnectionClass && !hasMultipleConnections) {
+            when {
+                isEnclaveConnection && delay == null -> """, $name{"$name", ${rightPorts[0].container.name}->__lf_env.get()}"""
+                isEnclaveConnection && delay != null -> """, $name{"$name", ${rightPorts[0].container.name}->__lf_env.get(), ${delay.toCppTime()}}"""
+                else                                 -> """, $name{"$name", this, ${delay.toCppTime()}}"""
             }
         } else null
+    }
 
-    private fun generateConstructorInitializer(connection: Connection): String? =
-        if (connection.requiresConnectionClass && !connection.hasMultipleConnections) {
-            val delay = connection.delay.toCppTime()
-            val name = connection.name
-            """, $name{"$name", this, $delay}"""
-        } else null
 }

org.lflang/src/org/lflang/generator/cpp/CppExtensions.kt

@@ -2,6 +2,7 @@ package org.lflang.generator.cpp
 
 import org.eclipse.emf.ecore.resource.Resource
 import org.lflang.*
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.isEnclave
 import org.lflang.lf.BuiltinTriggerRef
 import org.lflang.lf.Expression
 import org.lflang.lf.Port
@@ -111,8 +112,11 @@ val Reactor.templateName: String get() = if (isGeneric) "$name<${typeParms.joinT
 
 /** Get a C++ code representation of the given variable */
 val VarRef.name: String
-    get() = if (this.container == null) this.variable.name
-    else "${this.container.name}->${this.variable.name}"
+    get() = when {
+        container == null -> variable.name
+        container.isEnclave -> "${container.name}->__lf_instance->${variable.name}"
+        else -> "${container.name}->${variable.name}"
+    }
 
 /** Get a C++ code representation of the given trigger */
 val TriggerRef.name: String

org.lflang/src/org/lflang/generator/cpp/CppInstanceGenerator.kt

@@ -36,29 +36,49 @@ class CppInstanceGenerator(
     private val fileConfig: CppFileConfig,
     private val errorReporter: ErrorReporter
 ) {
-    private val Instantiation.isEnclave: Boolean get() = AttributeUtils.isEnclave(this)
+    companion object {
+        val Instantiation.isEnclave: Boolean get() = AttributeUtils.isEnclave(this)
 
-    private val Instantiation.hasEachParameter: Boolean
-        get() = AttributeUtils.getBooleanAttributeParameter(
-            AttributeUtils.getEnclaveAttribute(this), AttributeSpec.EACH_ATTR
-        ) ?: false
+        val Instantiation.hasEachParameter: Boolean
+            get() = AttributeUtils.getBooleanAttributeParameter(
+                AttributeUtils.getEnclaveAttribute(this), AttributeSpec.EACH_ATTR
+            ) ?: false
 
-    val Instantiation.cppType: String
-        get() {
-            return if (reactor.isGeneric)
+        private val Instantiation.reactorType: String
+            get() = if (reactor.isGeneric)
                 """${reactor.name}<${typeArgs.joinToString(", ") { it.toText() }}>"""
             else
                 reactor.name
-        }
+
+        val Instantiation.cppClass: String
+            get() = if (isEnclave) enclaveWrapperClassName else reactorType
+
+        val Instantiation.enclaveWrapperClassName get() = "EnclaveWrapper_$name"
+    }
+
+    private fun Instantiation.generateWrapper(): String = """
+            |struct $enclaveWrapperClassName {
+            |  ${if (!hasEachParameter) "inline static " else ""}std::unique_ptr<reactor::Environment> __lf_env{nullptr};
+            |  std::unique_ptr<$reactorType> __lf_instance{nullptr};
+            |  
+            |  $enclaveWrapperClassName(const std::string& name, reactor::Reactor* container, $reactorType::Parameters&& params) {
+            |    if (__lf_env == nullptr) {
+            |      __lf_env = std::make_unique<reactor::Environment>(container->fqn() + name, container->environment());
+            |    }
+            |    __lf_instance = std::make_unique<$reactorType>(name, __lf_env.get(), std::forward<$reactorType::Parameters>(params));
+            |  }
+            |};
+        """.trimMargin()
 
     private fun generateDeclaration(inst: Instantiation): String = with(inst) {
-        val instance = if (isBank) "std::vector<std::unique_ptr<$cppType>>" else "std::unique_ptr<$cppType>"
+        val instance = if (isBank) "std::vector<std::unique_ptr<$cppClass>>" else "std::unique_ptr<$cppClass>"
         if (isEnclave) {
-            val env = if (hasEachParameter) "std::vector<std::unique_ptr<reactor::Environment>>" else "reactor::Environment"
-            return """
-                $env __lf_env_$name;
-                $instance $name;
-            """.trimIndent()
+            return with(PrependOperator) {
+                """
+                ${" |"..inst.generateWrapper()}
+                    |$instance $name;
+                """.trimMargin()
+            }
         }
         return "$instance $name;"
     }
@@ -91,43 +111,25 @@ class CppInstanceGenerator(
         // by iterating over the reactor parameters we make sure that the parameters are assigned in declaration order
         return reactor.parameters.mapNotNull {
             if (it.name in assignments) ".${it.name} = ${assignments[it.name]}" else null
-        }.joinToString(", ", "$cppType::Parameters{", "}")
+        }.joinToString(", ", "$reactorType::Parameters{", "}")
     }
 
     private fun generateInitializer(inst: Instantiation): String? = with(inst) {
-        when {
-            !isBank && !isEnclave                    -> """, $name(std::make_unique<$cppType>("$name", this, ${getParameterStruct()}))"""
-            !isBank && isEnclave                     -> """
-                    , __lf_env_$name(this->fqn() + ".$name", this->environment())
-                    , $name(std::make_unique<$cppType>("$name", &__lf_env_$name, ${getParameterStruct()}))
-                """.trimIndent()
-
-            isBank && isEnclave && !hasEachParameter -> """, __lf_env_$name(this->fqn() + ".$name", this->environment())"""
-            else                                     -> null
-        }
+        if (isBank) null else """, $name(std::make_unique<$cppClass>("$name", this, ${getParameterStruct()}))"""
     }
 
     private fun generateConstructorInitializer(inst: Instantiation): String {
         with(inst) {
             assert(isBank)
-            val containerRef = when {
-                hasEachParameter -> "__lf_env_$name[__lf_idx].get()"
-                isEnclave        -> "&__lf_env_$name"
-                else             -> "this"
-            }
-
             val width = inst.widthSpec.toCppCode()
-            return with(PrependOperator) {
-                """
+            return """
                 |// initialize instance $name
                 |$name.reserve($width);
                 |for (size_t __lf_idx = 0; __lf_idx < $width; __lf_idx++) {
                 |  std::string __lf_inst_name = "${name}_" + std::to_string(__lf_idx);
-             ${"|  "..if (hasEachParameter) """__lf_env_$name.emplace_back(std::make_unique<reactor::Environment>(this->fqn() + "." + __lf_inst_name, this->environment()));""" else ""}
-                |  $name.emplace_back(std::make_unique<$cppType>(__lf_inst_name, $containerRef, ${inst.getParameterStruct()}));
+                |  $name.emplace_back(std::make_unique<$cppClass>(__lf_inst_name, this, ${inst.getParameterStruct()}));
                 |}
             """.trimMargin()
-            }
         }
     }
 

org.lflang/src/org/lflang/generator/cpp/CppPortGenerator.kt

@@ -24,6 +24,8 @@
 
 package org.lflang.generator.cpp
 
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.enclaveWrapperClassName
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.isEnclave
 import org.lflang.inferredType
 import org.lflang.isBank
 import org.lflang.isMultiport
@@ -34,7 +36,6 @@ class CppPortGenerator(private val reactor: Reactor) {
 
     companion object {
         private val VarRef.isMultiport get() = (variable as? Port)?.isMultiport == true
-        private val VarRef.isInBank get() = container?.isBank == true
 
         /**
          * Return the C++ type of a port reference.
@@ -44,11 +45,16 @@ class CppPortGenerator(private val reactor: Reactor) {
          * statement and let the C++ compiler do the job.
          */
         val VarRef.dataType: String
-            get() = when {
-                isInBank && isMultiport  -> "std::remove_reference<decltype(${container.name}[0]->${variable.name}[0])>::type::value_type"
-                isInBank && !isMultiport -> "std::remove_reference<decltype(${container.name}[0]->${variable.name})>::type::value_type"
-                !isInBank && isMultiport -> "std::remove_reference<decltype($name[0])>::type::value_type"
-                else                     -> "std::remove_reference<decltype($name)>::type::value_type"
+            get() {
+                val variableRef = when {
+                    this == null                             -> ""
+                    container == null                        -> this.name
+                    container.isBank && container.isEnclave  -> "${container.name}[0]->__lf_instance->${variable.name}"
+                    container.isBank && !container.isEnclave -> "${container.name}[0]->${variable.name}"
+                    else                                     -> this.name
+                }
+                val multiportRef = if (isMultiport) "$variableRef[0]" else variableRef
+                return "std::remove_reference<decltype($multiportRef)>::type::value_type"
             }
     }
 
@@ -62,7 +68,7 @@ class CppPortGenerator(private val reactor: Reactor) {
     }
 
     /** Get the C++ type for the receiving port. */
-    val Port.cppType: String
+    private val Port.cppType: String
         get() {
             val portType = when (this) {
                 is Input  -> "reactor::Input"

org.lflang/src/org/lflang/generator/cpp/CppReactionGenerator.kt

@@ -25,6 +25,7 @@
 package org.lflang.generator.cpp
 
 import org.lflang.generator.PrependOperator
+import org.lflang.generator.cpp.CppInstanceGenerator.Companion.cppClass
 import org.lflang.isBank
 import org.lflang.joinWithLn
 import org.lflang.label
@@ -172,7 +173,7 @@ class CppReactionGenerator(
     }
 
     private fun generateViewForContainer(r: Reaction, container: Instantiation): String {
-        val reactorClass = with(instanceGenerator) { container.cppType }
+        val reactorClass = container.cppClass
         val viewClass = r.getViewClassName(container)
         val viewInstance = r.getViewInstanceName(container)
 

org.lflang/src/org/lflang/generator/cpp/CppReactorGenerator.kt

@@ -187,8 +187,8 @@ class CppReactorGenerator(private val reactor: Reactor, fileConfig: CppFileConfi
             ${" |  "..instances.generateInitializers()}
             ${" |  "..timers.generateInitializers()}
             ${" |  "..actions.generateInitializers()}
-            ${" |  "..connections.generateInitializers()}
             ${" |  "..reactions.generateReactionViewInitializers()}
+            ${" |  "..connections.generateInitializers()}
                 |{
             ${" |  "..ports.generateConstructorInitializers()}
             ${" |  "..instances.generateConstructorInitializers()}