Merge pull request #1227 from mattjoyce/feature/template-extensions

Feature/template extensions

cli/cli.go

@@ -2,12 +2,13 @@ package cli
 
 import (
 	"fmt"
-	"github.com/danielmiessler/fabric/plugins/tools/youtube"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/danielmiessler/fabric/plugins/tools/youtube"
+
 	"github.com/danielmiessler/fabric/common"
 	"github.com/danielmiessler/fabric/core"
 	"github.com/danielmiessler/fabric/plugins/ai"
@@ -42,7 +43,10 @@ func Cli(version string) (err error) {
 		}
 	}
 
-	registry := core.NewPluginRegistry(fabricDb)
+	var registry *core.PluginRegistry
+	if registry, err = core.NewPluginRegistry(fabricDb); err != nil {
+			return
+	}
 
 	// if the setup flag is set, run the setup function
 	if currentFlags.Setup {
@@ -136,6 +140,23 @@ func Cli(version string) (err error) {
 		}
 	}
 
+if currentFlags.ListExtensions {
+    err = registry.TemplateExtensions.ListExtensions()
+    return
+}
+
+if currentFlags.AddExtension != "" {
+    err = registry.TemplateExtensions.RegisterExtension(currentFlags.AddExtension)
+    return
+}
+
+if currentFlags.RemoveExtension != "" {
+    err = registry.TemplateExtensions.RemoveExtension(currentFlags.RemoveExtension)
+    return
+}
+
+
+
 	// if the interactive flag is set, run the interactive function
 	// if currentFlags.Interactive {
 	// 	interactive.Interactive()

cli/flags.go

@@ -65,6 +65,10 @@ type Flags struct {
 	ServeAddress       string            `long:"address" description:"The address to bind the REST API" default:":8080"`
 	Config             string            `long:"config" description:"Path to YAML config file"`
 	Version            bool              `long:"version" description:"Print current version"`
+	ListExtensions     bool              `long:"listextensions" description:"List all registered extensions"`
+	AddExtension      string            `long:"addextension" description:"Register a new extension from config file path"`
+	RemoveExtension   string            `long:"rmextension" description:"Remove a registered extension by name"`
+
 }
 
 var debug = false

core/plugin_registry.go

@@ -3,6 +3,8 @@ package core
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strconv"
 
 	"github.com/samber/lo"
@@ -21,13 +23,14 @@ import (
 	"github.com/danielmiessler/fabric/plugins/ai/openrouter"
 	"github.com/danielmiessler/fabric/plugins/ai/siliconcloud"
 	"github.com/danielmiessler/fabric/plugins/db/fsdb"
+	"github.com/danielmiessler/fabric/plugins/template"
 	"github.com/danielmiessler/fabric/plugins/tools"
 	"github.com/danielmiessler/fabric/plugins/tools/jina"
 	"github.com/danielmiessler/fabric/plugins/tools/lang"
 	"github.com/danielmiessler/fabric/plugins/tools/youtube"
 )
 
-func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry) {
+func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry, err error) {
 	ret = &PluginRegistry{
 		Db:             db,
 		VendorManager:  ai.NewVendorsManager(),
@@ -37,6 +40,12 @@ func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry) {
 		Language:       lang.NewLanguage(),
 		Jina:           jina.NewClient(),
 	}
+
+	var homedir string
+	if homedir, err = os.UserHomeDir(); err != nil {
+		return
+	}
+	ret.TemplateExtensions = template.NewExtensionManager(filepath.Join(homedir, ".config/fabric"))
 
 	ret.Defaults = tools.NeeDefaults(ret.GetModels)
 
@@ -60,6 +69,7 @@ type PluginRegistry struct {
 	YouTube        *youtube.YouTube
 	Language       *lang.Language
 	Jina           *jina.Client
+	TemplateExtensions     *template.ExtensionManager
 }
 
 func (o *PluginRegistry) SaveEnvFile() (err error) {

core/plugin_registry_test.go

@@ -1,15 +1,20 @@
 package core
 
 import (
-	"github.com/danielmiessler/fabric/plugins/db/fsdb"
 	"os"
 	"testing"
+
+	"github.com/danielmiessler/fabric/plugins/db/fsdb"
 )
 
 func TestSaveEnvFile(t *testing.T) {
-	registry := NewPluginRegistry(fsdb.NewDb(os.TempDir()))
+	db := fsdb.NewDb(os.TempDir())
+	registry, err := NewPluginRegistry(db)
+	if err != nil {
+			t.Fatalf("NewPluginRegistry() error = %v", err)
+	}
 
-	err := registry.SaveEnvFile()
+	err = registry.SaveEnvFile()
 	if err != nil {
 		t.Fatalf("SaveEnvFile() error = %v", err)
 	}

plugins/template/Examples/README.md

@@ -0,0 +1,223 @@
+
+# Fabric Extensions: Complete Guide
+
+## Understanding Extension Architecture
+
+### Registry Structure
+The extension registry is stored at `~/.config/fabric/extensions/extensions.yaml` and tracks registered extensions:
+
+```yaml
+extensions:
+    extension-name:
+        config_path: /path/to/config.yaml
+        config_hash: <sha256>
+        executable_hash: <sha256>
+```
+
+The registry maintains security through hash verification of both configs and executables.
+
+### Extension Configuration
+Each extension requires a YAML configuration file with the following structure:
+
+```yaml
+name: "extension-name"          # Unique identifier
+executable: "/path/to/binary"   # Full path to executable
+type: "executable"             # Type of extension
+timeout: "30s"                 # Execution timeout
+description: "Description"     # What the extension does
+version: "1.0.0"              # Version number
+env: []                       # Optional environment variables
+
+operations:                   # Defined operations
+  operation-name:
+    cmd_template: "{{executable}} {{operation}} {{value}}"
+
+config:                      # Output configuration
+  output:
+    method: "stdout"         # or "file"
+    file_config:            # Optional, for file output
+      cleanup: true
+      path_from_stdout: true
+      work_dir: "/tmp"
+```
+
+### Directory Structure
+Recommended organization:
+```
+~/.config/fabric/extensions/
+├── bin/           # Extension executables
+├── configs/       # Extension YAML configs
+└── extensions.yaml # Registry file
+```
+
+## Example 1: Python Wrapper (Word Generator)
+A simple example wrapping a Python script.
+
+### 1. Position Files
+```bash
+# Create directories
+mkdir -p ~/.config/fabric/extensions/{bin,configs}
+
+# Install script
+cp word-generator.py ~/.config/fabric/extensions/bin/
+chmod +x ~/.config/fabric/extensions/bin/word-generator.py
+```
+
+### 2. Configure
+Create `~/.config/fabric/extensions/configs/word-generator.yaml`:
+```yaml
+name: word-generator
+executable: "~/.config/fabric/extensions/bin/word-generator.py"
+type: executable
+timeout: "5s"
+description: "Generates random words based on count parameter"
+version: "1.0.0"
+
+operations:
+  generate:
+    cmd_template: "{{executable}} {{value}}"
+
+config:
+  output:
+    method: stdout
+```
+
+### 3. Register & Run
+```bash
+# Register
+fabric --addextension ~/.config/fabric/extensions/configs/word-generator.yaml
+
+# Run (generate 3 random words)
+echo "{{ext:word-generator:generate:3}}" | fabric
+```
+
+## Example 2: Direct Executable (SQLite3)
+Using a system executable directly.
+
+copy the memories to your home directory
+ ~/memories.db
+
+### 1. Configure
+Create `~/.config/fabric/extensions/configs/memory-query.yaml`:
+```yaml
+name: memory-query
+executable: "/usr/bin/sqlite3"
+type: executable
+timeout: "5s"
+description: "Query memories database"
+version: "1.0.0"
+
+operations:
+  goal:
+    cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where type= 'goal'\""
+  value:
+    cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where type= 'value'\""
+  byid:
+    cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where uid= {{value}}\""
+  all:
+    cmd_template: "{{executable}} -json ~/memories.db \"select * from memories\""
+
+config:
+  output:
+    method: stdout
+```
+
+### 2. Register & Run
+```bash
+# Register
+fabric --addextension ~/.config/fabric/extensions/configs/memory-query.yaml
+
+# Run queries
+echo  "{{ext:memory-query:all}}" | fabric
+echo  "{{ext:memory-query:byid:3}}" | fabric
+```
+
+
+## Extension Management Commands
+
+### Add Extension
+```bash
+fabric --addextension ~/.config/fabric/extensions/configs/memory-query.yaml
+```
+
+Note : if the executable or config file changes, you must re-add the extension.
+This will recompute the hash for the extension.
+
+
+### List Extensions
+```bash
+fabric --listextensions
+```
+Shows all registered extensions with their status and configuration details.
+
+### Remove Extension
+```bash
+fabric --rmextension <extension-name>
+```
+Removes an extension from the registry.
+
+
+## Extensions in patterns
+
+```
+Create a pattern that use multiple extensions.
+
+These are my favorite
+{{ext:word-generator:generate:3}}
+
+These are my least favorite
+{{ext:word-generator:generate:2}}
+
+what does this say about me?
+```
+
+```bash
+./fabric -p ./plugins/template/Examples/test_pattern.md
+```
+
+## Security Considerations
+
+1. **Hash Verification**
+   - Both configs and executables are verified via SHA-256 hashes
+   - Changes to either require re-registration
+   - Prevents tampering with registered extensions
+
+2. **Execution Safety**
+   - Extensions run with user permissions
+   - Timeout constraints prevent runaway processes
+   - Environment variables can be controlled via config
+
+3. **Best Practices**
+   - Review extension code before installation
+   - Keep executables in protected directories
+   - Use absolute paths in configurations
+   - Implement proper error handling in scripts
+   - Regular security audits of registered extensions
+
+## Troubleshooting
+
+### Common Issues
+1. **Registration Failures**
+   - Verify file permissions
+   - Check executable paths
+   - Validate YAML syntax
+
+2. **Execution Errors**
+   - Check operation exists in config
+   - Verify timeout settings
+   - Monitor system resources
+   - Check extension logs
+
+3. **Output Issues**
+   - Verify output method configuration
+   - Check file permissions for file output
+   - Monitor disk space for file operations
+
+### Debug Tips
+1. Enable verbose logging when available
+2. Check system logs for execution errors
+3. Verify extension dependencies
+4. Test extensions with minimal configurations first
+
+
+Would you like me to expand on any particular section or add more examples?

plugins/template/Examples/remote-security-report.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# remote-security-report.sh
+# Usage: remote-security-report.sh cert host [report_name]
+
+cert_path="$1"
+host="$2"
+report_name="${3:-report}"
+temp_file="/tmp/security-report-${report_name}.txt"
+
+# Copy the security report script to remote host
+scp -i "$cert_path" /usr/local/bin/security-report.sh "${host}:~/security-report.sh" >&2
+
+# Make it executable and run it on remote host
+ssh -i "$cert_path" "$host" "chmod +x ~/security-report.sh && sudo ~/security-report.sh ${temp_file}" >&2
+
+# Copy the report back
+scp -i "$cert_path" "${host}:${temp_file}" "${temp_file}" >&2
+
+# Cleanup remote files
+ssh -i "$cert_path" "$host" "rm ~/security-report.sh ${temp_file}" >&2
+
+# Output the local file path for fabric to read
+echo "${temp_file}"
+

plugins/template/Examples/remote-security-report.yaml

@@ -0,0 +1,17 @@
+name: "remote-security"
+executable: "/usr/local/bin/remote-security-report.sh"
+type: "executable"
+timeout: "60s"
+description: "Generate security report from remote system"
+
+operations:
+  report:
+    cmd_template: "{{executable}} {{1}} {{2}} {{3}}"
+
+config:
+  output:
+    method: "file"
+    file_config:
+      cleanup: true
+      path_from_stdout: true
+      work_dir: "/tmp"