diff --git a/README.md b/README.md index 0b60e1a75..1d2284096 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ > [!NOTE] > November 8, 2024 > -> - **Multimodal Support**: You can now us `-a` (attachment) for Multimodal submissions to OpenAI models that support it. Example: `fabric -a https://path/to/image "Give me a description of this image."` +> - **Multimodal Support**: You can now use `-a` (attachment) for Multimodal submissions to OpenAI models that support it. Example: `fabric -a https://path/to/image "Give me a description of this image."` ## What and why diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000..8bf4fd331 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,68 @@ +# YAML Configuration Support + +## Overview +Fabric now supports YAML configuration files for commonly used options. This allows users to persist settings and share configurations across multiple runs. + +## Usage +Use the `--config` flag to specify a YAML configuration file: +```bash +fabric --config ~/.config/fabric/config.yaml "Tell me about APIs" +``` + +## Configuration Precedence +1. CLI flags (highest priority) +2. YAML config values +3. Default values (lowest priority) + +## Supported Configuration Options +```yaml +# Model selection +model: gpt-4 +modelContextLength: 4096 + +# Model parameters +temperature: 0.7 +topp: 0.9 +presencepenalty: 0.0 +frequencypenalty: 0.0 +seed: 42 + +# Pattern selection +pattern: analyze # Use pattern name or filename + +# Feature flags +stream: true +raw: false +``` + +## Rules and Behavior +- Only long flag names are supported in YAML (e.g., `temperature` not `-t`) +- CLI flags always override YAML values +- Unknown YAML declarations are ignored +- If a declaration appears multiple times in YAML, the last one wins +- The order of YAML declarations doesn't matter + +## Type Conversions +The following string-to-type conversions are supported: +- String to number: `"42"` → `42` +- String to float: `"42.5"` → `42.5` +- String to boolean: `"true"` → `true` + +## Example Config +```yaml +# ~/.config/fabric/config.yaml +model: gpt-4 +temperature: 0.8 +pattern: analyze +stream: true +topp: 0.95 +presencepenalty: 0.1 +frequencypenalty: 0.2 +``` + +## CLI Override Example +```bash +# Override temperature from config +fabric --config ~/.config/fabric/config.yaml --temperature 0.9 "Query" +``` + diff --git a/cli/cli.go b/cli/cli.go index 5614f5043..81c0e0d3f 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -56,6 +56,12 @@ func Cli(version string) (err error) { return } + if currentFlags.ServeOllama { + registry.ConfigureVendors() + err = restapi.ServeOllama(registry, currentFlags.ServeAddress, version) + return + } + if currentFlags.UpdatePatterns { err = registry.PatternsLoader.PopulateDB() return diff --git a/cli/example.yaml b/cli/example.yaml new file mode 100644 index 000000000..d9d61c429 --- /dev/null +++ b/cli/example.yaml @@ -0,0 +1,21 @@ +#this is an example yaml config file for fabric + +# use fabric pattern names +pattern: ai + +# or use a filename +# pattern: ~/testpattern.md + +model: phi3:latest + +# for models that support context length +modelContextLength: 2048 + +frequencypenalty: 0.5 +presencepenalty: 0.5 +topp: 0.67 +temperature: 0.88 +seed: 42 + +stream: true +raw: false \ No newline at end of file diff --git a/cli/flags.go b/cli/flags.go index 552d8b5e9..15603c4ea 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -7,6 +7,7 @@ import ( "io" "os" "reflect" + "strconv" "strings" "github.com/jessevdk/go-flags" @@ -60,6 +61,7 @@ type Flags struct { InputHasVars bool `long:"input-has-vars" description:"Apply variables to user input"` DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"` Serve bool `long:"serve" description:"Serve the Fabric Rest API"` + ServeOllama bool `long:"serveOllama" description:"Serve the Fabric Rest API with ollama endpoints"` ServeAddress string `long:"address" description:"The address to bind the REST API" default:":8080"` Config string `long:"config" description:"Path to YAML config file"` Version bool `long:"version" description:"Print current version"` @@ -77,7 +79,7 @@ func Debugf(format string, a ...interface{}) { func Init() (ret *Flags, err error) { // Track which yaml-configured flags were set on CLI usedFlags := make(map[string]bool) - args := os.Args[1:] + yamlArgsScan := os.Args[1:] // Get list of fields that have yaml tags, could be in yaml config yamlFields := make(map[string]bool) @@ -90,7 +92,7 @@ func Init() (ret *Flags, err error) { } // Scan args for that are provided by cli and might be in yaml - for _, arg := range args { + for _, arg := range yamlArgsScan { if strings.HasPrefix(arg, "--") { flag := strings.TrimPrefix(arg, "--") if i := strings.Index(flag, "="); i > 0 { @@ -106,7 +108,8 @@ func Init() (ret *Flags, err error) { // Parse CLI flags first ret = &Flags{} parser := flags.NewParser(ret, flags.Default) - if _, err = parser.Parse(); err != nil { + var args []string + if args, err = parser.Parse(); err != nil { return nil, err } @@ -148,6 +151,7 @@ func Init() (ret *Flags, err error) { info, _ := os.Stdin.Stat() pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0 + // Append positional arguments to the message (custom message) if len(args) > 0 { ret.Message = AppendMessage(ret.Message, args[len(args)-1]) } @@ -164,38 +168,36 @@ func Init() (ret *Flags, err error) { } func assignWithConversion(targetField, sourceField reflect.Value) error { - switch targetField.Kind() { - case reflect.Float64: - if sourceField.Kind() == reflect.Int || sourceField.Kind() == reflect.Float32 { - targetField.SetFloat(float64(sourceField.Convert(reflect.TypeOf(float64(0))).Float())) - Debugf("Converted field %s : %v\n", targetField.Type(), targetField.Interface()) - return nil - } - case reflect.Int: - if sourceField.Kind() == reflect.Float64 || sourceField.Kind() == reflect.Float32 { - targetField.SetInt(int64(sourceField.Convert(reflect.TypeOf(int64(0))).Int())) - Debugf("Converted field %s : %v\n", targetField.Type(), targetField.Interface()) - return nil - } - case reflect.String: - if sourceField.Kind() == reflect.Interface { - if str, ok := sourceField.Interface().(string); ok { - targetField.SetString(str) - Debugf("Converted field %s : %v\n", targetField.Type(), targetField.Interface()) + // Handle string source values + if sourceField.Kind() == reflect.String { + str := sourceField.String() + switch targetField.Kind() { + case reflect.Int: + // Try parsing as float first to handle "42.9" -> 42 + if val, err := strconv.ParseFloat(str, 64); err == nil { + targetField.SetInt(int64(val)) return nil } - } - case reflect.Bool: - if sourceField.Kind() == reflect.Interface { - if b, ok := sourceField.Interface().(bool); ok { - targetField.SetBool(b) - Debugf("Converted field %s : %v\n", targetField.Type(), targetField.Interface()) + // Try direct int parse + if val, err := strconv.ParseInt(str, 10, 64); err == nil { + targetField.SetInt(val) + return nil + } + case reflect.Float64: + if val, err := strconv.ParseFloat(str, 64); err == nil { + targetField.SetFloat(val) + return nil + } + case reflect.Bool: + if val, err := strconv.ParseBool(str); err == nil { + targetField.SetBool(val) return nil } } + return fmt.Errorf("cannot convert string %q to %v", str, targetField.Kind()) } - return fmt.Errorf("unsupported conversion: %s to %s", sourceField.Type(), targetField.Type()) + return fmt.Errorf("unsupported conversion from %v to %v", sourceField.Kind(), targetField.Kind()) } func loadYAMLConfig(configPath string) (*Flags, error) { diff --git a/patterns/analyze_answers/README.md b/patterns/analyze_answers/README.md index 92250b354..a1d3019a8 100644 --- a/patterns/analyze_answers/README.md +++ b/patterns/analyze_answers/README.md @@ -26,11 +26,11 @@ Subject: Machine Learning ``` -# Example run un bash: +# Example run bash: Copy the input query to the clipboard and execute the following command: -``` bash +```bash xclip -selection clipboard -o | fabric -sp analize_answers ``` diff --git a/patterns/create_quiz/README.md b/patterns/create_quiz/README.md index 0d6eb220b..a319f74cd 100644 --- a/patterns/create_quiz/README.md +++ b/patterns/create_quiz/README.md @@ -1,6 +1,6 @@ # Learning questionnaire generation -This pattern generates questions to help a learner/student review the main concepts of the learning objectives provided. +This pattern generates questions to help a learner/student review the main concepts of the learning objectives provided. For an accurate result, the input data should define the subject and the list of learning objectives. @@ -17,11 +17,11 @@ Learning Objectives: * Define unsupervised learning ``` -# Example run un bash: +# Example run bash: Copy the input query to the clipboard and execute the following command: -``` bash +```bash xclip -selection clipboard -o | fabric -sp create_quiz ``` diff --git a/patterns/summarize_paper/README.md b/patterns/summarize_paper/README.md index 1394cf7b0..99f906de5 100644 --- a/patterns/summarize_paper/README.md +++ b/patterns/summarize_paper/README.md @@ -21,19 +21,19 @@ This pattern generates a summary of an academic paper based on the provided text Copy the paper text to the clipboard and execute the following command: -``` bash +```bash pbpaste | fabric --pattern summarize_paper ``` or -``` bash +```bash pbpaste | summarize_paper ``` # Example output: -``` markdown +```markdown ### Title and authors of the Paper: **Internet of Paint (IoP): Channel Modeling and Capacity Analysis for Terahertz Electromagnetic Nanonetworks Embedded in Paint** Authors: Lasantha Thakshila Wedage, Mehmet C. Vuran, Bernard Butler, Yevgeni Koucheryavy, Sasitharan Balasubramaniam diff --git a/patterns/translate/system.md b/patterns/translate/system.md index 8dc0da586..858e1cc23 100644 --- a/patterns/translate/system.md +++ b/patterns/translate/system.md @@ -8,7 +8,7 @@ Take a step back, and breathe deeply and think step by step about how to achieve - The original format of the input must remain intact. -- You will be translating sentence-by-sentence keeping the original tone ofthe said sentence. +- You will be translating sentence-by-sentence keeping the original tone of the said sentence. - You will not be manipulate the wording to change the meaning. diff --git a/pkgs/fabric/version.nix b/pkgs/fabric/version.nix index 15801c64e..475a7f0cb 100644 --- a/pkgs/fabric/version.nix +++ b/pkgs/fabric/version.nix @@ -1 +1 @@ -"..1" +"1.4.124" \ No newline at end of file diff --git a/restapi/ollama.go b/restapi/ollama.go new file mode 100644 index 000000000..0d806729a --- /dev/null +++ b/restapi/ollama.go @@ -0,0 +1,275 @@ +package restapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/danielmiessler/fabric/core" + "github.com/gin-gonic/gin" + "io" + "log" + "net/http" + "strings" + "time" +) + +type OllamaModel struct { + Models []Model `json:"models"` +} +type Model struct { + Details ModelDetails `json:"details"` + Digest string `json:"digest"` + Model string `json:"model"` + ModifiedAt string `json:"modified_at"` + Name string `json:"name"` + Size int64 `json:"size"` +} + +type ModelDetails struct { + Families []string `json:"families"` + Family string `json:"family"` + Format string `json:"format"` + ParameterSize string `json:"parameter_size"` + ParentModel string `json:"parent_model"` + QuantizationLevel string `json:"quantization_level"` +} + +type APIConvert struct { + registry *core.PluginRegistry + r *gin.Engine + addr *string +} + +type OllamaRequestBody struct { + Messages []OllamaMessage `json:"messages"` + Model string `json:"model"` + Options struct { + } `json:"options"` + Stream bool `json:"stream"` +} + +type OllamaMessage struct { + Content string `json:"content"` + Role string `json:"role"` +} + +type OllamaResponse struct { + Model string `json:"model"` + CreatedAt string `json:"created_at"` + Message struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"message"` + DoneReason string `json:"done_reason,omitempty"` + Done bool `json:"done"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` +} + +type FabricResponseFormat struct { + Type string `json:"type"` + Format string `json:"format"` + Content string `json:"content"` +} + +func ServeOllama(registry *core.PluginRegistry, address string, version string) (err error) { + r := gin.New() + + // Middleware + r.Use(gin.Logger()) + r.Use(gin.Recovery()) + + // Register routes + fabricDb := registry.Db + NewPatternsHandler(r, fabricDb.Patterns) + NewContextsHandler(r, fabricDb.Contexts) + NewSessionsHandler(r, fabricDb.Sessions) + NewChatHandler(r, registry, fabricDb) + NewConfigHandler(r, fabricDb) + NewModelsHandler(r, registry.VendorManager) + + typeConversion := APIConvert{ + registry: registry, + r: r, + addr: &address, + } + // Ollama Endpoints + r.GET("/api/tags", typeConversion.ollamaTags) + r.GET("/api/version", func(c *gin.Context) { + c.Data(200, "application/json", []byte(fmt.Sprintf("{\"%s\"}", version))) + return + }) + r.POST("/api/chat", typeConversion.ollamaChat) + + // Start server + err = r.Run(address) + if err != nil { + return err + } + + return +} + +func (f APIConvert) ollamaTags(c *gin.Context) { + patterns, err := f.registry.Db.Patterns.GetNames() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err}) + return + } + var response OllamaModel + for _, pattern := range patterns { + today := time.Now().Format("2024-11-25T12:07:58.915991813-05:00") + details := ModelDetails{ + Families: []string{"fabric"}, + Family: "fabric", + Format: "custom", + ParameterSize: "42.0B", + ParentModel: "", + QuantizationLevel: "", + } + response.Models = append(response.Models, Model{ + Details: details, + Digest: "365c0bd3c000a25d28ddbf732fe1c6add414de7275464c4e4d1c3b5fcb5d8ad1", + Model: fmt.Sprintf("%s:latest", pattern), + ModifiedAt: today, + Name: fmt.Sprintf("%s:latest", pattern), + Size: 0, + }) + } + + c.JSON(200, response) + +} + +func (f APIConvert) ollamaChat(c *gin.Context) { + body, err := io.ReadAll(c.Request.Body) + if err != nil { + log.Printf("Error reading body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "testing endpoint"}) + return + } + var prompt OllamaRequestBody + err = json.Unmarshal(body, &prompt) + if err != nil { + log.Printf("Error unmarshalling body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "testing endpoint"}) + return + } + now := time.Now() + var chat ChatRequest + + if len(prompt.Messages) == 1 { + chat.Prompts = []PromptRequest{{ + UserInput: prompt.Messages[0].Content, + Vendor: "", + Model: "", + ContextName: "", + PatternName: strings.Split(prompt.Model, ":")[0], + }} + } else if len(prompt.Messages) > 1 { + var content string + for _, msg := range prompt.Messages { + content = fmt.Sprintf("%s%s:%s\n", content, msg.Role, msg.Content) + } + chat.Prompts = []PromptRequest{{ + UserInput: content, + Vendor: "", + Model: "", + ContextName: "", + PatternName: strings.Split(prompt.Model, ":")[0], + }} + } + fabricChatReq, err := json.Marshal(chat) + if err != nil { + log.Printf("Error marshalling body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err}) + return + } + ctx := context.Background() + var req *http.Request + if strings.Contains(*f.addr, "http") { + req, err = http.NewRequest("POST", fmt.Sprintf("%s/chat", *f.addr), bytes.NewBuffer(fabricChatReq)) + } else { + req, err = http.NewRequest("POST", fmt.Sprintf("http://127.0.0.1%s/chat", *f.addr), bytes.NewBuffer(fabricChatReq)) + } + if err != nil { + log.Fatal(err) + } + + req = req.WithContext(ctx) + + fabricRes, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("Error getting /chat body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err}) + return + } + body, err = io.ReadAll(fabricRes.Body) + if err != nil { + log.Printf("Error reading body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "testing endpoint"}) + return + } + var forwardedResponse OllamaResponse + var forwardedResponses []OllamaResponse + var fabricResponse FabricResponseFormat + err = json.Unmarshal([]byte(strings.Split(strings.Split(string(body), "\n")[0], "data: ")[1]), &fabricResponse) + if err != nil { + log.Printf("Error unmarshalling body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "testing endpoint"}) + return + } + for _, word := range strings.Split(fabricResponse.Content, " ") { + forwardedResponse = OllamaResponse{ + Model: "", + CreatedAt: "", + Message: struct { + Role string `json:"role"` + Content string `json:"content"` + }(struct { + Role string + Content string + }{Content: fmt.Sprintf("%s ", word), Role: "assistant"}), + Done: false, + } + forwardedResponses = append(forwardedResponses, forwardedResponse) + } + forwardedResponse.Model = prompt.Model + forwardedResponse.CreatedAt = time.Now().UTC().Format("2006-01-02T15:04:05.999999999Z") + forwardedResponse.Message.Role = "assistant" + forwardedResponse.Message.Content = "" + forwardedResponse.DoneReason = "stop" + forwardedResponse.Done = true + forwardedResponse.TotalDuration = time.Since(now).Nanoseconds() + forwardedResponse.LoadDuration = int(time.Since(now).Nanoseconds()) + forwardedResponse.PromptEvalCount = 42 + forwardedResponse.PromptEvalDuration = int(time.Since(now).Nanoseconds()) + forwardedResponse.EvalCount = 420 + forwardedResponse.EvalDuration = time.Since(now).Nanoseconds() + forwardedResponses = append(forwardedResponses, forwardedResponse) + + var res []byte + for _, response := range forwardedResponses { + marshalled, err := json.Marshal(response) + if err != nil { + log.Printf("Error marshalling body: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err}) + return + } + for _, bytein := range marshalled { + res = append(res, bytein) + } + for _, bytebreak := range []byte("\n") { + res = append(res, bytebreak) + } + } + c.Data(200, "application/json", res) + + //c.JSON(200, forwardedResponse) + return +} diff --git a/version.go b/version.go index 49affa658..e244e7e50 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package main -var version = "v..1" +var version = "v1.4.124" \ No newline at end of file