From bde1f8b61e31790b8127f19d9d6c241b8b280122 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 09:47:58 +0100 Subject: [PATCH 01/10] Remove TrackExecution function --- logging/db.go | 8 -------- logging/process.go | 5 ----- queries/queries.go | 25 ------------------------- 3 files changed, 38 deletions(-) diff --git a/logging/db.go b/logging/db.go index ae483e5f..fa0849a8 100644 --- a/logging/db.go +++ b/logging/db.go @@ -273,14 +273,6 @@ func (logDB *LoggerDB) CleanQueryLogs(entries int64) error { if err := logDB.Database.Conn.Unscoped().Delete(&queriesTargets).Error; err != nil { return err } - // Get query executions - var queriesExecutions []queries.DistributedQueryExecution - if err := logDB.Database.Conn.Where("name = ?", q.Name).Find(&queriesExecutions).Error; err != nil { - return err - } - if err := logDB.Database.Conn.Unscoped().Delete(&queriesExecutions).Error; err != nil { - return err - } // Delete query if err := logDB.Database.Conn.Unscoped().Delete(&q).Error; err != nil { return err diff --git a/logging/process.go b/logging/process.go index 82eebd3e..51a01d9d 100644 --- a/logging/process.go +++ b/logging/process.go @@ -81,11 +81,6 @@ func (l *LoggerTLS) ProcessLogQueryResult(queriesWrite types.QueryWriteRequest, if err != nil { log.Err(err).Msg("error updating query") } - // TODO: This TrackExeuction need be removed - // Add a record for this query - if err := l.Queries.TrackExecution(q, node.UUID, queriesWrite.Statuses[q]); err != nil { - log.Err(err).Msg("error adding query execution") - } // Instead of creating a new record in a separate table, we can just update the query status if err := l.Queries.UpdateQueryStatus(q, node.ID, queriesWrite.Statuses[q]); err != nil { log.Err(err).Msg("error updating query status") diff --git a/queries/queries.go b/queries/queries.go index 3d613962..051c4003 100644 --- a/queries/queries.go +++ b/queries/queries.go @@ -102,14 +102,6 @@ type DistributedQueryTarget struct { Value string } -// DistributedQueryExecution to keep track of queries executing -type DistributedQueryExecution struct { - gorm.Model - Name string `gorm:"index"` - UUID string `gorm:"index"` - Result int -} - // QueryReadQueries to hold all the on-demand queries type QueryReadQueries map[string]string @@ -131,10 +123,6 @@ func CreateQueries(backend *gorm.DB) *Queries { if err := backend.AutoMigrate(&DistributedQuery{}); err != nil { log.Fatal().Msgf("Failed to AutoMigrate table (distributed_queries): %v", err) } - // table distributed_query_executions - if err := backend.AutoMigrate(&DistributedQueryExecution{}); err != nil { - log.Fatal().Msgf("Failed to AutoMigrate table (distributed_query_executions): %v", err) - } // table distributed_query_targets if err := backend.AutoMigrate(&DistributedQueryTarget{}); err != nil { log.Fatal().Msgf("Failed to AutoMigrate table (distributed_query_targets): %v", err) @@ -505,19 +493,6 @@ func (q *Queries) UpdateQueryStatus(queryName string, nodeID uint, statusCode in return nil } -// TrackExecution to keep track of where queries have already ran -func (q *Queries) TrackExecution(name, uuid string, result int) error { - queryExecution := DistributedQueryExecution{ - Name: name, - UUID: uuid, - Result: result, - } - if err := q.DB.Create(&queryExecution).Error; err != nil { - return err - } - return nil -} - // Helper to decide whether if the query targets apply to a give node func isQueryTarget(node nodes.OsqueryNode, targets []DistributedQueryTarget) bool { for _, t := range targets { From 7f635a407cb568db7f10484f64628f6a8171378d Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 09:48:51 +0100 Subject: [PATCH 02/10] Remove NotYetExecuted function --- queries/queries.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/queries/queries.go b/queries/queries.go index 051c4003..73383f31 100644 --- a/queries/queries.go +++ b/queries/queries.go @@ -421,13 +421,6 @@ func (q *Queries) GetTargets(name string) ([]DistributedQueryTarget, error) { return targets, nil } -// NotYetExecuted to check if query already executed or it is within the interval -func (q *Queries) NotYetExecuted(name, uuid string) bool { - var results int64 - q.DB.Model(&DistributedQueryExecution{}).Where("name = ? AND uuid = ?", name, uuid).Count(&results) - return (results == 0) -} - // IncExecution to increase the execution count for this query func (q *Queries) IncExecution(name string, envid uint) error { query, err := q.Get(name, envid) From 9f6b6c02c063bf6e7593f433251d5a4ed5964da7 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 09:49:58 +0100 Subject: [PATCH 03/10] Remove isQueryTarget as not be used --- queries/queries.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/queries/queries.go b/queries/queries.go index 73383f31..16d7118b 100644 --- a/queries/queries.go +++ b/queries/queries.go @@ -485,26 +485,3 @@ func (q *Queries) UpdateQueryStatus(queryName string, nodeID uint, statusCode in } return nil } - -// Helper to decide whether if the query targets apply to a give node -func isQueryTarget(node nodes.OsqueryNode, targets []DistributedQueryTarget) bool { - for _, t := range targets { - // Check for environment match - if t.Type == QueryTargetEnvironment && t.Value == node.Environment { - return true - } - // Check for platform match - if t.Type == QueryTargetPlatform && node.Platform == t.Value { - return true - } - // Check for UUID match - if t.Type == QueryTargetUUID && node.UUID == t.Value { - return true - } - // Check for localname match - if t.Type == QueryTargetLocalname && node.Localname == t.Value { - return true - } - } - return false -} From 40672a8c803b241e5121a4ce6b90bebaa9fc9f23 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 10:36:16 +0100 Subject: [PATCH 04/10] Replace VerifyComplete function --- admin/main.go | 8 ++++++-- logging/process.go | 6 +----- queries/queries.go | 31 +++++++++++++++++-------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/admin/main.go b/admin/main.go index 16b52f11..3c5826fb 100644 --- a/admin/main.go +++ b/admin/main.go @@ -767,10 +767,14 @@ func osctrlAdminService() { log.Err(err).Msg("Error getting all environments") } for _, e := range allEnvs { - if err:= queriesmgr.CleanupExpiredQueries(e.ID); err != nil { + // try to complete the queries first + if err := queriesmgr.CleanupCompletedQueries(e.ID); err != nil { + log.Err(err).Msg("Error completing expired queries") + } + if err := queriesmgr.CleanupExpiredQueries(e.ID); err != nil { log.Err(err).Msg("Error cleaning up expired queries") } - if err:= queriesmgr.CleanupExpiredCarves(e.ID); err != nil { + if err := queriesmgr.CleanupExpiredCarves(e.ID); err != nil { log.Err(err).Msg("Error cleaning up expired carves") } } diff --git a/logging/process.go b/logging/process.go index 51a01d9d..f1d47f79 100644 --- a/logging/process.go +++ b/logging/process.go @@ -81,13 +81,9 @@ func (l *LoggerTLS) ProcessLogQueryResult(queriesWrite types.QueryWriteRequest, if err != nil { log.Err(err).Msg("error updating query") } - // Instead of creating a new record in a separate table, we can just update the query status + // Update query status if err := l.Queries.UpdateQueryStatus(q, node.ID, queriesWrite.Statuses[q]); err != nil { log.Err(err).Msg("error updating query status") } - // Check if query is completed - if err := l.Queries.VerifyComplete(q, envid); err != nil { - log.Err(err).Msg("error verifying and completing query") - } } } diff --git a/queries/queries.go b/queries/queries.go index 16d7118b..3862fb27 100644 --- a/queries/queries.go +++ b/queries/queries.go @@ -297,20 +297,6 @@ func (q *Queries) Complete(name string, envid uint) error { return nil } -// VerifyComplete to mark query as completed if the expected executions are done -func (q *Queries) VerifyComplete(name string, envid uint) error { - query, err := q.Get(name, envid) - if err != nil { - return err - } - if (query.Executions + query.Errors) >= query.Expected { - if err := q.DB.Model(&query).Updates(map[string]interface{}{"completed": true, "active": false}).Error; err != nil { - return err - } - } - return nil -} - // Activate to mark query as active func (q *Queries) Activate(name string, envid uint) error { query, err := q.Get(name, envid) @@ -347,6 +333,23 @@ func (q *Queries) Expire(name string, envid uint) error { return nil } +// CleanupCompletedQueries to set all completed queries as inactive by environment +func (q *Queries) CleanupCompletedQueries(envid uint) error { + qs, err := q.GetQueries(TargetActive, envid) + if err != nil { + return err + } + for _, query := range qs { + executionReached := (query.Executions + query.Errors) >= query.Expected + if executionReached { + if err := q.DB.Model(&query).Updates(map[string]interface{}{"completed": true, "active": false}).Error; err != nil { + return err + } + } + } + return nil +} + // CleanupExpiredQueries to set all expired queries as inactive by environment func (q *Queries) CleanupExpiredQueries(envid uint) error { qs, err := q.GetQueries(TargetActive, envid) From b93e2e419cd6fbf0a02da7cf879d74883963ceb8 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 11:55:23 +0100 Subject: [PATCH 05/10] Create node queries in batch --- api/handlers/queries.go | 11 +++++++++-- queries/queries.go | 20 +++++++++++++------- queries/queries_test.go | 35 +++++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 37b7c9e4..1a11d77e 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -226,14 +226,21 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) expectedClear := removeStringDuplicates(expected) // Create new record for query list + nodesID := make([]uint, len(expectedClear)) for _, nodeUUID := range expectedClear { node, err := h.Nodes.GetByUUID(nodeUUID) if err != nil { log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) continue } - if err := h.Queries.CreateNodeQuery(node.ID, newQuery.ID); err != nil { - log.Err(err).Msgf("error creating node query for query %s and node %s", newQuery.Name, nodeUUID) + nodesID = append(nodesID, node.ID) + } + // If the list is empty, we don't need to create node queries + if len(nodesID) != 0 { + if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { + log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) + apiErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) + return } } diff --git a/queries/queries.go b/queries/queries.go index 3862fb27..e0b7275f 100644 --- a/queries/queries.go +++ b/queries/queries.go @@ -390,13 +390,19 @@ func (q *Queries) Create(query DistributedQuery) error { return nil } -// CreateNodeQuery to link a node to a query -func (q *Queries) CreateNodeQuery(nodeID, queryID uint) error { - nodeQuery := NodeQuery{ - NodeID: nodeID, - QueryID: queryID, - } - if err := q.DB.Create(&nodeQuery).Error; err != nil { +// CreateNodeQueries to link multiple nodes to a query +func (q *Queries) CreateNodeQueries(nodeIDs []uint, queryID uint) error { + if len(nodeIDs) == 0 { + return fmt.Errorf("no nodes to link to query") + } + var nodeQueries []NodeQuery + for _, nodeID := range nodeIDs { + nodeQueries = append(nodeQueries, NodeQuery{ + NodeID: nodeID, + QueryID: queryID, + }) + } + if err := q.DB.Create(&nodeQueries).Error; err != nil { return err } return nil diff --git a/queries/queries_test.go b/queries/queries_test.go index cb1e642a..528a3d56 100644 --- a/queries/queries_test.go +++ b/queries/queries_test.go @@ -131,7 +131,8 @@ func TestUpdateQueryStatus(t *testing.T) { assert.Equal(t, queries.DistributedQueryStatusCompleted, updatedNodeQuery.Status, "Status does not match expected value") } -func TestCreateNodeQuery(t *testing.T) { + +func TestCreateNodeQueries(t *testing.T) { db, err := setupTestDB() if err != nil { t.Fatalf("Failed to setup test database: %v", err) @@ -142,9 +143,12 @@ func TestCreateNodeQuery(t *testing.T) { nodes.CreateNodes(db) // Create test data - node := nodes.OsqueryNode{ + node1 := nodes.OsqueryNode{ Model: gorm.Model{ID: 1}, } + node2 := nodes.OsqueryNode{ + Model: gorm.Model{ID: 2}, + } distributedQuery := queries.DistributedQuery{ Model: gorm.Model{ID: 1}, Name: "test_query", @@ -153,24 +157,31 @@ func TestCreateNodeQuery(t *testing.T) { Expiration: time.Now().Add(24 * time.Hour), } - if err := db.Create(&node).Error; err != nil { - t.Fatalf("Failed to create test node: %v", err) + if err := db.Create(&node1).Error; err != nil { + t.Fatalf("Failed to create test node1: %v", err) + } + if err := db.Create(&node2).Error; err != nil { + t.Fatalf("Failed to create test node2: %v", err) } if err := db.Create(&distributedQuery).Error; err != nil { t.Fatalf("Failed to create test distributed query: %v", err) } - // Test CreateNodeQuery function - err = q.CreateNodeQuery(1, 1) + // Test CreateNodeQueries function + nodeIDs := []uint{1, 2} + err = q.CreateNodeQueries(nodeIDs, 1) if err != nil { - t.Fatalf("CreateNodeQuery returned an error: %v", err) + t.Fatalf("CreateNodeQueries returned an error: %v", err) } - var nodeQuery queries.NodeQuery - if err := db.Where("node_id = ? AND query_id = ?", 1, 1).Find(&nodeQuery).Error; err != nil { - t.Fatalf("Failed to find created node query: %v", err) + var nodeQueries []queries.NodeQuery + if err := db.Where("query_id = ?", 1).Find(&nodeQueries).Error; err != nil { + t.Fatalf("Failed to find created node queries: %v", err) } - assert.Equal(t, uint(1), nodeQuery.NodeID, "NodeID does not match expected value") - assert.Equal(t, uint(1), nodeQuery.QueryID, "QueryID does not match expected value") + assert.Len(t, nodeQueries, 2, "Expected 2 node queries to be created") + assert.Equal(t, uint(1), nodeQueries[0].NodeID, "First NodeID does not match expected value") + assert.Equal(t, uint(1), nodeQueries[0].QueryID, "First QueryID does not match expected value") + assert.Equal(t, uint(2), nodeQueries[1].NodeID, "Second NodeID does not match expected value") + assert.Equal(t, uint(1), nodeQueries[1].QueryID, "Second QueryID does not match expected value") } From 6c261f333f52d2ea5519b43c70b233451cce8a29 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 13:09:28 +0100 Subject: [PATCH 06/10] add batch creation for admin --- admin/handlers/post.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/admin/handlers/post.go b/admin/handlers/post.go index 44bb151d..502bfb7c 100644 --- a/admin/handlers/post.go +++ b/admin/handlers/post.go @@ -231,14 +231,21 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque expectedClear := removeStringDuplicates(expected) // Create new record for query list + nodesID := make([]uint, len(expectedClear)) for _, nodeUUID := range expectedClear { node, err := h.Nodes.GetByUUID(nodeUUID) if err != nil { log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) continue } - if err := h.Queries.CreateNodeQuery(node.ID, newQuery.ID); err != nil { - log.Err(err).Msgf("error creating node query for query %s and node %s", newQuery.Name, nodeUUID) + nodesID = append(nodesID, node.ID) + } + // If the list is empty, we don't need to create node queries + if len(nodesID) != 0 { + if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { + log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) + adminErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) + return } } // Update value for expected From 370977b34bdd3aee431bf453f951479000d20251 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 14:09:28 +0100 Subject: [PATCH 07/10] Update the way of creating distributed query in osctrl-api --- api/handlers/queries.go | 77 ++++++++++++++++++----------------------- nodes/nodes.go | 2 +- utils/utils.go | 25 +++++++++++++ utils/utils_test.go | 35 ++++++++++++++----- 4 files changed, 85 insertions(+), 54 deletions(-) diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 1a11d77e..16ab6919 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -150,17 +150,17 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) return } - // Temporary list of UUIDs to calculate Expected - var expected []string - // Create targets + // List all the nodes that match the query + var expected []uint + + targetNodesID := []uint{} + // Current logic is to select nodes meeting all criteria in the query + // TODO: I believe we should only allow to list nodes in one environment in URL paths + // We will refactor this part to be tag based queries and add more options to the query if len(q.Environments) > 0 { + expected = []uint{} for _, e := range q.Environments { if (e != "") && h.Envs.Exists(e) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetEnvironment, e); err != nil { - apiErrorResponse(w, "error creating query environment target", http.StatusInternalServerError, err) - h.Inc(metricAPIQueriesErr) - return - } nodes, err := h.Nodes.GetByEnv(e, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { apiErrorResponse(w, "error getting nodes by environment", http.StatusInternalServerError, err) @@ -168,21 +168,18 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create platform target if len(q.Platforms) > 0 { + expected = []uint{} platforms, _ := h.Nodes.GetAllPlatforms() for _, p := range q.Platforms { if (p != "") && checkValidPlatform(platforms, p) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetPlatform, p); err != nil { - apiErrorResponse(w, "error creating query platform target", http.StatusInternalServerError, err) - h.Inc(metricAPIQueriesErr) - return - } nodes, err := h.Nodes.GetByPlatform(p, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { apiErrorResponse(w, "error getting nodes by platform", http.StatusInternalServerError, err) @@ -190,54 +187,46 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create UUIDs target if len(q.UUIDs) > 0 { + expected = []uint{} for _, u := range q.UUIDs { - if (u != "") && h.Nodes.CheckByUUID(u) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetUUID, u); err != nil { - apiErrorResponse(w, "error creating query UUID target", http.StatusInternalServerError, err) - h.Inc(metricAPIQueriesErr) - return + if u != "" { + node, err := h.Nodes.GetByUUID(u) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) + continue } - expected = append(expected, u) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target if len(q.Hosts) > 0 { - for _, _h := range q.Hosts { - if (_h != "") && h.Nodes.CheckByHost(_h) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetLocalname, _h); err != nil { - apiErrorResponse(w, "error creating query hostname target", http.StatusInternalServerError, err) - h.Inc(metricAPIQueriesErr) - return + expected = []uint{} + for _, hostName := range q.Hosts { + if hostName != "" { + node, err := h.Nodes.GetByIdentifier(hostName) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", hostName) + continue } - expected = append(expected, _h) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } - // Remove duplicates from expected - expectedClear := removeStringDuplicates(expected) - - // Create new record for query list - nodesID := make([]uint, len(expectedClear)) - for _, nodeUUID := range expectedClear { - node, err := h.Nodes.GetByUUID(nodeUUID) - if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) - continue - } - nodesID = append(nodesID, node.ID) - } // If the list is empty, we don't need to create node queries - if len(nodesID) != 0 { - if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { + if len(targetNodesID) != 0 { + if err := h.Queries.CreateNodeQueries(targetNodesID, newQuery.ID); err != nil { log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) apiErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) return @@ -245,7 +234,7 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) } // Update value for expected - if err := h.Queries.SetExpected(queryName, len(expectedClear), env.ID); err != nil { + if err := h.Queries.SetExpected(queryName, len(targetNodesID), env.ID); err != nil { apiErrorResponse(w, "error setting expected", http.StatusInternalServerError, err) h.Inc(metricAPICarvesErr) return diff --git a/nodes/nodes.go b/nodes/nodes.go index 4bfe99ac..a327a6ca 100644 --- a/nodes/nodes.go +++ b/nodes/nodes.go @@ -347,7 +347,7 @@ func (n *NodeManager) UpdateMetadataByUUID(uuid string, metadata NodeMetadata) e return fmt.Errorf("RecordUsername %v", err) } if metadata.Username != node.Username && metadata.Username != "" { - updates["username"] =metadata.Username + updates["username"] = metadata.Username } // Record hostname if err := n.RecordHostname(metadata.Hostname, node); err != nil { diff --git a/utils/utils.go b/utils/utils.go index 4a15b669..9127d61f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -31,3 +31,28 @@ func RandomForNames() string { _, _ = hasher.Write([]byte(fmt.Sprintf("%x", b))) return hex.EncodeToString(hasher.Sum(nil)) } + +func Intersect(slice1, slice2 []uint) []uint { + if len(slice1) == 0 { + return slice2 + } + // If slice2 is empty, return slice1 + if len(slice2) == 0 { + return slice1 + } + + set := make(map[uint]struct{}) + for _, item := range slice1 { + set[item] = struct{}{} // Add items from slice1 to the set + } + + intersection := []uint{} + for _, item := range slice2 { + if _, exists := set[item]; exists { + intersection = append(intersection, item) + delete(set, item) // Ensure uniqueness in the result + } + } + + return intersection +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 9cfa196c..9c9e35f6 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,27 +1,44 @@ -package utils +package utils_test import ( "testing" + "github.com/jmpsec/osctrl/utils" "github.com/stretchr/testify/assert" ) func TestBytesReceivedConversionBytes(t *testing.T) { - assert.NotEmpty(t, BytesReceivedConversion(123)) - assert.Equal(t, "123 bytes", BytesReceivedConversion(123)) + assert.NotEmpty(t, utils.BytesReceivedConversion(123)) + assert.Equal(t, "123 bytes", utils.BytesReceivedConversion(123)) } func TestBytesReceivedConversionKBytes(t *testing.T) { - assert.NotEmpty(t, BytesReceivedConversion(1024)) - assert.Equal(t, "1.0 KB", BytesReceivedConversion(1024)) + assert.NotEmpty(t, utils.BytesReceivedConversion(1024)) + assert.Equal(t, "1.0 KB", utils.BytesReceivedConversion(1024)) } func TestBytesReceivedConversionMBytes(t *testing.T) { - assert.NotEmpty(t, BytesReceivedConversion(1048576)) - assert.Equal(t, "1.0 MB", BytesReceivedConversion(1048576)) + assert.NotEmpty(t, utils.BytesReceivedConversion(1048576)) + assert.Equal(t, "1.0 MB", utils.BytesReceivedConversion(1048576)) } func TestRandomForNames(t *testing.T) { - assert.NotEmpty(t, RandomForNames()) - assert.Equal(t, 32, len(RandomForNames())) + assert.NotEmpty(t, utils.RandomForNames()) + assert.Equal(t, 32, len(utils.RandomForNames())) +} + +func TestIntersect(t *testing.T) { + var slice1 = []uint{1, 2, 3, 4, 5} + var slice2 = []uint{3, 4, 5, 6, 7} + var expected = []uint{3, 4, 5} + assert.Equal(t, expected, utils.Intersect(slice1, slice2)) + slice1 = utils.Intersect(slice1, slice2) + assert.Equal(t, expected, slice1) +} + +func TestIntersectEmpty(t *testing.T) { + var slice1 = []uint{} + var slice2 = []uint{3, 4, 5, 6, 7} + var expected = []uint{3, 4, 5, 6, 7} + assert.Equal(t, expected, utils.Intersect(slice1, slice2)) } From 234e42e697d41066c45ad1e55f3b2fe4810292cb Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 16:12:15 +0100 Subject: [PATCH 08/10] Update the query for osctrl-admin --- admin/handlers/post.go | 72 ++++++++++++++++++----------------------- api/handlers/queries.go | 6 ++-- api/handlers/utils.go | 15 --------- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/admin/handlers/post.go b/admin/handlers/post.go index 502bfb7c..7322b2a3 100644 --- a/admin/handlers/post.go +++ b/admin/handlers/post.go @@ -156,17 +156,17 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque adminErrorResponse(w, "error creating query", http.StatusInternalServerError, err) return } - // Temporary list of UUIDs to calculate Expected - var expected []string + + // List all the nodes that match the query + var expected []uint + + targetNodesID := []uint{} + // TODO: Refactor this to use osctrl-api instead of direct DB queries // Create environment target if len(q.Environments) > 0 { + expected = []uint{} for _, e := range q.Environments { if (e != "") && h.Envs.Exists(e) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetEnvironment, e); err != nil { - adminErrorResponse(w, "error creating query environment target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return - } nodes, err := h.Nodes.GetByEnv(e, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by environment", http.StatusInternalServerError, err) @@ -174,21 +174,18 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create platform target if len(q.Platforms) > 0 { + expected = []uint{} platforms, _ := h.Nodes.GetAllPlatforms() for _, p := range q.Platforms { if (p != "") && checkValidPlatform(platforms, p) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetPlatform, p); err != nil { - adminErrorResponse(w, "error creating query platform target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return - } nodes, err := h.Nodes.GetByPlatform(p, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by platform", http.StatusInternalServerError, err) @@ -196,60 +193,53 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create UUIDs target if len(q.UUIDs) > 0 { + expected = []uint{} for _, u := range q.UUIDs { - if (u != "") && h.Nodes.CheckByUUID(u) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetUUID, u); err != nil { - adminErrorResponse(w, "error creating query UUID target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return + if u != "" { + node, err := h.Nodes.GetByUUID(u) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) + continue } - expected = append(expected, u) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target if len(q.Hosts) > 0 { + expected = []uint{} for _, _h := range q.Hosts { - if (_h != "") && h.Nodes.CheckByHost(_h) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetLocalname, _h); err != nil { - adminErrorResponse(w, "error creating query hostname target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return + if _h != "" { + node, err := h.Nodes.GetByIdentifier(_h) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", _h) + continue } - expected = append(expected, _h) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } - // Remove duplicates from expected - expectedClear := removeStringDuplicates(expected) - // Create new record for query list - nodesID := make([]uint, len(expectedClear)) - for _, nodeUUID := range expectedClear { - node, err := h.Nodes.GetByUUID(nodeUUID) - if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) - continue - } - nodesID = append(nodesID, node.ID) - } // If the list is empty, we don't need to create node queries - if len(nodesID) != 0 { - if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { + if len(targetNodesID) != 0 { + if err := h.Queries.CreateNodeQueries(targetNodesID, newQuery.ID); err != nil { log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) adminErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) return } } // Update value for expected - if err := h.Queries.SetExpected(newQuery.Name, len(expectedClear), env.ID); err != nil { + if err := h.Queries.SetExpected(newQuery.Name, len(targetNodesID), env.ID); err != nil { adminErrorResponse(w, "error setting expected", http.StatusInternalServerError, err) h.Inc(metricAdminErr) return diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 16ab6919..75dcd03a 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -200,7 +200,7 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) if u != "" { node, err := h.Nodes.GetByUUID(u) if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) + log.Warn().Msgf("error getting node %s and failed to create node query for it", u) continue } expected = append(expected, node.ID) @@ -209,13 +209,15 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target + // Currently we are using the GetByIdentifier function and it need be more clear + // about the definition of the identifier if len(q.Hosts) > 0 { expected = []uint{} for _, hostName := range q.Hosts { if hostName != "" { node, err := h.Nodes.GetByIdentifier(hostName) if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", hostName) + log.Warn().Msgf("error getting node %s and failed to create node query for it", hostName) continue } expected = append(expected, node.ID) diff --git a/api/handlers/utils.go b/api/handlers/utils.go index 2c4f63cf..0a642e67 100644 --- a/api/handlers/utils.go +++ b/api/handlers/utils.go @@ -53,18 +53,3 @@ func checkValidPlatform(platforms []string, platform string) bool { } return false } - -// Helper to remove duplicates from []string -func removeStringDuplicates(s []string) []string { - seen := make(map[string]struct{}, len(s)) - i := 0 - for _, v := range s { - if _, ok := seen[v]; ok { - continue - } - seen[v] = struct{}{} - s[i] = v - i++ - } - return s[:i] -} From b7234581469699b73777d82b293bcdd6f9037022 Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Wed, 11 Dec 2024 16:26:49 +0100 Subject: [PATCH 09/10] Revert "Update the query for osctrl-admin" This reverts commit 234e42e697d41066c45ad1e55f3b2fe4810292cb. --- admin/handlers/post.go | 72 +++++++++++++++++++++++------------------ api/handlers/queries.go | 6 ++-- api/handlers/utils.go | 15 +++++++++ 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/admin/handlers/post.go b/admin/handlers/post.go index 7322b2a3..502bfb7c 100644 --- a/admin/handlers/post.go +++ b/admin/handlers/post.go @@ -156,17 +156,17 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque adminErrorResponse(w, "error creating query", http.StatusInternalServerError, err) return } - - // List all the nodes that match the query - var expected []uint - - targetNodesID := []uint{} - // TODO: Refactor this to use osctrl-api instead of direct DB queries + // Temporary list of UUIDs to calculate Expected + var expected []string // Create environment target if len(q.Environments) > 0 { - expected = []uint{} for _, e := range q.Environments { if (e != "") && h.Envs.Exists(e) { + if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetEnvironment, e); err != nil { + adminErrorResponse(w, "error creating query environment target", http.StatusInternalServerError, err) + h.Inc(metricAdminErr) + return + } nodes, err := h.Nodes.GetByEnv(e, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by environment", http.StatusInternalServerError, err) @@ -174,18 +174,21 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.ID) + expected = append(expected, n.UUID) } } } - targetNodesID = utils.Intersect(targetNodesID, expected) } // Create platform target if len(q.Platforms) > 0 { - expected = []uint{} platforms, _ := h.Nodes.GetAllPlatforms() for _, p := range q.Platforms { if (p != "") && checkValidPlatform(platforms, p) { + if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetPlatform, p); err != nil { + adminErrorResponse(w, "error creating query platform target", http.StatusInternalServerError, err) + h.Inc(metricAdminErr) + return + } nodes, err := h.Nodes.GetByPlatform(p, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by platform", http.StatusInternalServerError, err) @@ -193,53 +196,60 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.ID) + expected = append(expected, n.UUID) } } } - targetNodesID = utils.Intersect(targetNodesID, expected) } // Create UUIDs target if len(q.UUIDs) > 0 { - expected = []uint{} for _, u := range q.UUIDs { - if u != "" { - node, err := h.Nodes.GetByUUID(u) - if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) - continue + if (u != "") && h.Nodes.CheckByUUID(u) { + if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetUUID, u); err != nil { + adminErrorResponse(w, "error creating query UUID target", http.StatusInternalServerError, err) + h.Inc(metricAdminErr) + return } - expected = append(expected, node.ID) + expected = append(expected, u) } } - targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target if len(q.Hosts) > 0 { - expected = []uint{} for _, _h := range q.Hosts { - if _h != "" { - node, err := h.Nodes.GetByIdentifier(_h) - if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", _h) - continue + if (_h != "") && h.Nodes.CheckByHost(_h) { + if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetLocalname, _h); err != nil { + adminErrorResponse(w, "error creating query hostname target", http.StatusInternalServerError, err) + h.Inc(metricAdminErr) + return } - expected = append(expected, node.ID) + expected = append(expected, _h) } } - targetNodesID = utils.Intersect(targetNodesID, expected) } + // Remove duplicates from expected + expectedClear := removeStringDuplicates(expected) + // Create new record for query list + nodesID := make([]uint, len(expectedClear)) + for _, nodeUUID := range expectedClear { + node, err := h.Nodes.GetByUUID(nodeUUID) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) + continue + } + nodesID = append(nodesID, node.ID) + } // If the list is empty, we don't need to create node queries - if len(targetNodesID) != 0 { - if err := h.Queries.CreateNodeQueries(targetNodesID, newQuery.ID); err != nil { + if len(nodesID) != 0 { + if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) adminErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) return } } // Update value for expected - if err := h.Queries.SetExpected(newQuery.Name, len(targetNodesID), env.ID); err != nil { + if err := h.Queries.SetExpected(newQuery.Name, len(expectedClear), env.ID); err != nil { adminErrorResponse(w, "error setting expected", http.StatusInternalServerError, err) h.Inc(metricAdminErr) return diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 75dcd03a..16ab6919 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -200,7 +200,7 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) if u != "" { node, err := h.Nodes.GetByUUID(u) if err != nil { - log.Warn().Msgf("error getting node %s and failed to create node query for it", u) + log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) continue } expected = append(expected, node.ID) @@ -209,15 +209,13 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target - // Currently we are using the GetByIdentifier function and it need be more clear - // about the definition of the identifier if len(q.Hosts) > 0 { expected = []uint{} for _, hostName := range q.Hosts { if hostName != "" { node, err := h.Nodes.GetByIdentifier(hostName) if err != nil { - log.Warn().Msgf("error getting node %s and failed to create node query for it", hostName) + log.Err(err).Msgf("error getting node %s and failed to create node query for it", hostName) continue } expected = append(expected, node.ID) diff --git a/api/handlers/utils.go b/api/handlers/utils.go index 0a642e67..2c4f63cf 100644 --- a/api/handlers/utils.go +++ b/api/handlers/utils.go @@ -53,3 +53,18 @@ func checkValidPlatform(platforms []string, platform string) bool { } return false } + +// Helper to remove duplicates from []string +func removeStringDuplicates(s []string) []string { + seen := make(map[string]struct{}, len(s)) + i := 0 + for _, v := range s { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + s[i] = v + i++ + } + return s[:i] +} From a741aa86aa4272d2a154b2a87a118d44a2a046be Mon Sep 17 00:00:00 2001 From: Zhuoyuan Liu Date: Thu, 12 Dec 2024 09:29:05 +0100 Subject: [PATCH 10/10] Revert "Revert "Update the query for osctrl-admin"" This reverts commit b7234581469699b73777d82b293bcdd6f9037022. --- admin/handlers/post.go | 72 ++++++++++++++++++----------------------- api/handlers/queries.go | 6 ++-- api/handlers/utils.go | 15 --------- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/admin/handlers/post.go b/admin/handlers/post.go index 502bfb7c..7322b2a3 100644 --- a/admin/handlers/post.go +++ b/admin/handlers/post.go @@ -156,17 +156,17 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque adminErrorResponse(w, "error creating query", http.StatusInternalServerError, err) return } - // Temporary list of UUIDs to calculate Expected - var expected []string + + // List all the nodes that match the query + var expected []uint + + targetNodesID := []uint{} + // TODO: Refactor this to use osctrl-api instead of direct DB queries // Create environment target if len(q.Environments) > 0 { + expected = []uint{} for _, e := range q.Environments { if (e != "") && h.Envs.Exists(e) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetEnvironment, e); err != nil { - adminErrorResponse(w, "error creating query environment target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return - } nodes, err := h.Nodes.GetByEnv(e, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by environment", http.StatusInternalServerError, err) @@ -174,21 +174,18 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create platform target if len(q.Platforms) > 0 { + expected = []uint{} platforms, _ := h.Nodes.GetAllPlatforms() for _, p := range q.Platforms { if (p != "") && checkValidPlatform(platforms, p) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetPlatform, p); err != nil { - adminErrorResponse(w, "error creating query platform target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return - } nodes, err := h.Nodes.GetByPlatform(p, "active", h.Settings.InactiveHours(settings.NoEnvironmentID)) if err != nil { adminErrorResponse(w, "error getting nodes by platform", http.StatusInternalServerError, err) @@ -196,60 +193,53 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque return } for _, n := range nodes { - expected = append(expected, n.UUID) + expected = append(expected, n.ID) } } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create UUIDs target if len(q.UUIDs) > 0 { + expected = []uint{} for _, u := range q.UUIDs { - if (u != "") && h.Nodes.CheckByUUID(u) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetUUID, u); err != nil { - adminErrorResponse(w, "error creating query UUID target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return + if u != "" { + node, err := h.Nodes.GetByUUID(u) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) + continue } - expected = append(expected, u) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target if len(q.Hosts) > 0 { + expected = []uint{} for _, _h := range q.Hosts { - if (_h != "") && h.Nodes.CheckByHost(_h) { - if err := h.Queries.CreateTarget(newQuery.Name, queries.QueryTargetLocalname, _h); err != nil { - adminErrorResponse(w, "error creating query hostname target", http.StatusInternalServerError, err) - h.Inc(metricAdminErr) - return + if _h != "" { + node, err := h.Nodes.GetByIdentifier(_h) + if err != nil { + log.Err(err).Msgf("error getting node %s and failed to create node query for it", _h) + continue } - expected = append(expected, _h) + expected = append(expected, node.ID) } } + targetNodesID = utils.Intersect(targetNodesID, expected) } - // Remove duplicates from expected - expectedClear := removeStringDuplicates(expected) - // Create new record for query list - nodesID := make([]uint, len(expectedClear)) - for _, nodeUUID := range expectedClear { - node, err := h.Nodes.GetByUUID(nodeUUID) - if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", nodeUUID) - continue - } - nodesID = append(nodesID, node.ID) - } // If the list is empty, we don't need to create node queries - if len(nodesID) != 0 { - if err := h.Queries.CreateNodeQueries(nodesID, newQuery.ID); err != nil { + if len(targetNodesID) != 0 { + if err := h.Queries.CreateNodeQueries(targetNodesID, newQuery.ID); err != nil { log.Err(err).Msgf("error creating node queries for query %s", newQuery.Name) adminErrorResponse(w, "error creating node queries", http.StatusInternalServerError, err) return } } // Update value for expected - if err := h.Queries.SetExpected(newQuery.Name, len(expectedClear), env.ID); err != nil { + if err := h.Queries.SetExpected(newQuery.Name, len(targetNodesID), env.ID); err != nil { adminErrorResponse(w, "error setting expected", http.StatusInternalServerError, err) h.Inc(metricAdminErr) return diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 16ab6919..75dcd03a 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -200,7 +200,7 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) if u != "" { node, err := h.Nodes.GetByUUID(u) if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", u) + log.Warn().Msgf("error getting node %s and failed to create node query for it", u) continue } expected = append(expected, node.ID) @@ -209,13 +209,15 @@ func (h *HandlersApi) QueriesRunHandler(w http.ResponseWriter, r *http.Request) targetNodesID = utils.Intersect(targetNodesID, expected) } // Create hostnames target + // Currently we are using the GetByIdentifier function and it need be more clear + // about the definition of the identifier if len(q.Hosts) > 0 { expected = []uint{} for _, hostName := range q.Hosts { if hostName != "" { node, err := h.Nodes.GetByIdentifier(hostName) if err != nil { - log.Err(err).Msgf("error getting node %s and failed to create node query for it", hostName) + log.Warn().Msgf("error getting node %s and failed to create node query for it", hostName) continue } expected = append(expected, node.ID) diff --git a/api/handlers/utils.go b/api/handlers/utils.go index 2c4f63cf..0a642e67 100644 --- a/api/handlers/utils.go +++ b/api/handlers/utils.go @@ -53,18 +53,3 @@ func checkValidPlatform(platforms []string, platform string) bool { } return false } - -// Helper to remove duplicates from []string -func removeStringDuplicates(s []string) []string { - seen := make(map[string]struct{}, len(s)) - i := 0 - for _, v := range s { - if _, ok := seen[v]; ok { - continue - } - seen[v] = struct{}{} - s[i] = v - i++ - } - return s[:i] -}