diff --git a/ee/server/service/setup_experience.go b/ee/server/service/setup_experience.go index 79698a67e28d..7a75bca9738d 100644 --- a/ee/server/service/setup_experience.go +++ b/ee/server/service/setup_experience.go @@ -178,6 +178,7 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string case len(installersPending) > 0: // enqueue installers for _, installer := range installersPending { + // TODO(mna): this should be top priority as this is setup exp. installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, false, nil) if err != nil { return false, ctxerr.Wrap(ctx, err, "queueing setup experience install request") diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index bfe732426e87..99b342d283c8 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -248,6 +248,9 @@ func (ds *Datastore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { // NOTE: Be sure to update both the count (here) and list statements (below) // if the query condition is modified. + + // TODO(mna): use the upcoming queue instead (but also include in-execution stuff + // that have no results yet)? countStmts := []string{ `SELECT COUNT(*) c @@ -299,6 +302,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint // NOTE: Be sure to update both the count (above) and list statements (below) // if the query condition is modified. + + // TODO(mna): use the upcoming queue instead (but also include in-execution stuff + // that have no results yet)? listStmts := []string{ // list pending scripts `SELECT @@ -436,15 +442,15 @@ SELECT ) AS details FROM host_vpp_software_installs hvsi -INNER JOIN +INNER JOIN nano_view_queue nvq ON nvq.command_uuid = hvsi.command_uuid -LEFT OUTER JOIN +LEFT OUTER JOIN users u ON hvsi.user_id = u.id -LEFT OUTER JOIN +LEFT OUTER JOIN host_display_names hdn ON hdn.host_id = hvsi.host_id -LEFT OUTER JOIN +LEFT OUTER JOIN vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform -LEFT OUTER JOIN +LEFT OUTER JOIN software_titles st ON st.id = vpa.title_id WHERE nvq.status IS NULL diff --git a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go index 8f5c3e73542e..1100b8196500 100644 --- a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go @@ -2,6 +2,7 @@ package tables import ( "database/sql" + "fmt" ) func init() { @@ -11,26 +12,52 @@ func init() { func Up_20250106162751(tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE upcoming_activities ( - id INT UNSIGNED NOT NULL AUTO_INCREMENT, - host_id INT UNSIGNED NOT NULL, + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + host_id INT UNSIGNED NOT NULL, -- priority 0 is normal, > 0 is higher priority, < 0 is lower priority. - priority INT NOT NULL DEFAULT 0, + priority INT NOT NULL DEFAULT 0, -- user_id is the user that triggered the activity, it may be null if the -- activity is fleet-initiated or the user was deleted. Additional user -- information (name, email, etc.) is stored in the JSON payload. - user_id INT UNSIGNED NULL, + user_id INT UNSIGNED NULL, + fleet_initiated TINYINT(1) NOT NULL DEFAULT 0, -- type of activity to be executed, currently we only support those, but as -- more activity types get added, we can enrich the ENUM with an ALTER TABLE. - activity_type ENUM('script', 'software_install', 'vpp_app_install') NOT NULL, + activity_type ENUM('script', 'software_install', 'software_uninstall', 'vpp_app_install') NOT NULL, -- execution_id is the identifier of the activity that will be used when -- executed - e.g. scripts and software installs have an execution_id, and -- it is sometimes important to know it as soon as the activity is enqueued, - -- so we need to generate it immediately. + -- so we need to generate it immediately. Every activity will be identified + -- via this unique execution_id. execution_id VARCHAR(255) NOT NULL, + payload JSON NOT NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (id), + UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), + -- index for the common access pattern to get the next activity to execute + INDEX idx_upcoming_activities_host_id_priority_created_at (host_id, priority, created_at), + -- index for the common access pattern to get by activity type (e.g. deleting pending scripts) + INDEX idx_upcoming_activities_host_id_activity_type (activity_type, host_id), + CONSTRAINT fk_upcoming_activities_user_id + FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return fmt.Errorf("failed to create upcoming_activities: %w", err) + } + + _, err = tx.Exec(` +CREATE TABLE script_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, -- those are all columns and not JSON fields because we need FKs on them to -- do processing ON DELETE, otherwise we'd have to check for existence of @@ -41,28 +68,60 @@ CREATE TABLE upcoming_activities ( policy_id INT UNSIGNED NULL, setup_experience_script_id INT UNSIGNED NULL, - payload JSON NOT NULL, - -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues created_at DATETIME(6) NOT NULL DEFAULT NOW(6), updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), - PRIMARY KEY (id), - UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), - INDEX idx_upcoming_activities_host_id_activity_type (host_id, priority, created_at, activity_type), - CONSTRAINT fk_upcoming_activities_script_id + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_script_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_script_upcoming_activities_script_id FOREIGN KEY (script_id) REFERENCES scripts (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_script_content_id + CONSTRAINT fk_script_upcoming_activities_script_content_id FOREIGN KEY (script_content_id) REFERENCES script_contents (id) ON DELETE CASCADE, - CONSTRAINT fk_upcoming_activities_policy_id + CONSTRAINT fk_script_upcoming_activities_policy_id FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_setup_experience_script_id - FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_user_id - FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL -) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`, + CONSTRAINT fk_script_upcoming_activities_setup_experience_script_id + FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, ) - return err + if err != nil { + return err + } + + _, err = tx.Exec(` +CREATE TABLE software_install_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, + + -- those are all columns and not JSON fields because we need FKs on them to + -- do processing ON DELETE, otherwise we'd have to check for existence of + -- each one when executing the activity (we need the enqueue next activity + -- action to be efficient). + software_installer_id INT UNSIGNED NULL, + policy_id INT UNSIGNED NULL, + software_title_id INT UNSIGNED NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_software_install_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_software_installer_id + FOREIGN KEY (software_installer_id) REFERENCES software_installers (id) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_policy_id + FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, + CONSTRAINT fk_software_install_upcoming_activities_software_title_id + FOREIGN KEY (software_title_id) REFERENCES software_titles (id) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return err + } + return nil } func Down_20250106162751(tx *sql.Tx) error { diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index fce32fea06b2..381d16af1ace 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1644,6 +1644,28 @@ CREATE TABLE `script_contents` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `script_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `script_id` int unsigned DEFAULT NULL, + `script_content_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `setup_experience_script_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_script_upcoming_activities_script_id` (`script_id`), + KEY `fk_script_upcoming_activities_script_content_id` (`script_content_id`), + KEY `fk_script_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_script_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`), + CONSTRAINT `fk_script_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_script_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `scripts` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `team_id` int unsigned DEFAULT NULL, @@ -1797,6 +1819,25 @@ CREATE TABLE `software_host_counts` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `software_install_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `software_installer_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `software_title_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_software_install_upcoming_activities_software_installer_id` (`software_installer_id`), + KEY `fk_software_install_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_software_install_upcoming_activities_software_title_id` (`software_title_id`), + CONSTRAINT `fk_software_install_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_software_install_upcoming_activities_software_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_software_title_id` FOREIGN KEY (`software_title_id`) REFERENCES `software_titles` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `software_installer_labels` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `software_installer_id` int unsigned NOT NULL, @@ -1912,31 +1953,21 @@ CREATE TABLE `teams` ( /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `upcoming_activities` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, + `id` bigint unsigned NOT NULL AUTO_INCREMENT, `host_id` int unsigned NOT NULL, `priority` int NOT NULL DEFAULT '0', `user_id` int unsigned DEFAULT NULL, - `activity_type` enum('script','software_install','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, + `fleet_initiated` tinyint(1) NOT NULL DEFAULT '0', + `activity_type` enum('script','software_install','software_uninstall','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - `script_id` int unsigned DEFAULT NULL, - `script_content_id` int unsigned DEFAULT NULL, - `policy_id` int unsigned DEFAULT NULL, - `setup_experience_script_id` int unsigned DEFAULT NULL, `payload` json NOT NULL, `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`id`), UNIQUE KEY `idx_upcoming_activities_execution_id` (`execution_id`), - KEY `idx_upcoming_activities_host_id_activity_type` (`host_id`,`priority`,`created_at`,`activity_type`), - KEY `fk_upcoming_activities_script_id` (`script_id`), - KEY `fk_upcoming_activities_script_content_id` (`script_content_id`), - KEY `fk_upcoming_activities_policy_id` (`policy_id`), - KEY `fk_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`), + KEY `idx_upcoming_activities_host_id_priority_created_at` (`host_id`,`priority`,`created_at`), + KEY `idx_upcoming_activities_host_id_activity_type` (`activity_type`,`host_id`), KEY `fk_upcoming_activities_user_id` (`user_id`), - CONSTRAINT `fk_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, - CONSTRAINT `fk_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL, - CONSTRAINT `fk_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL, CONSTRAINT `fk_upcoming_activities_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 6118f3786675..e1f24bc7faf3 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -62,14 +62,11 @@ func (ds *Datastore) NewInternalScriptExecutionRequest(ctx context.Context, requ func newHostScriptExecutionRequest(ctx context.Context, tx sqlx.ExtContext, request *fleet.HostScriptRequestPayload, isInternal bool) (*fleet.HostScriptResult, error) { const ( - insStmt = ` + insUAStmt = ` INSERT INTO upcoming_activities - ( - host_id, user_id, activity_type, execution_id, script_id, script_content_id, - policy_id, setup_experience_script_id, priority, payload - ) + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) VALUES - (?, ?, 'script', ?, ?, ?, ?, ?, ?, + (?, ?, ?, ?, 'script', ?, JSON_OBJECT( 'sync_request', ?, 'is_internal', ?, @@ -77,15 +74,24 @@ VALUES ) )` + insSUAStmt = ` +INSERT INTO script_upcoming_activities + (upcoming_activity_id, script_id, script_content_id, policy_id, setup_experience_script_id) +VALUES + (?, ?, ?, ?, ?) +` + getStmt = ` SELECT - ua.id, ua.host_id, ua.execution_id, ua.created_at, ua.script_id, ua.policy_id, ua.user_id, + ua.id, ua.host_id, ua.execution_id, ua.created_at, sua.script_id, sua.policy_id, ua.user_id, JSON_EXTRACT(payload, '$.sync_request') AS sync_request, - sc.contents as script_contents, ua.setup_experience_script_id + sc.contents as script_contents, sua.setup_experience_script_id FROM upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id INNER JOIN script_contents sc - ON ua.script_content_id = sc.id + ON sua.script_content_id = sc.id WHERE ua.id = ? ` @@ -99,28 +105,36 @@ WHERE } execID := uuid.New().String() - result, err := tx.ExecContext(ctx, insStmt, + result, err := tx.ExecContext(ctx, insUAStmt, request.HostID, + priority, request.UserID, + false, // TODO(mna): do we have fleet-initiated scripts? execID, + request.SyncRequest, + isInternal, + request.UserID, + ) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "new script upcoming activity") + } + + activityID, _ := result.LastInsertId() + _, err = tx.ExecContext(ctx, insSUAStmt, + activityID, request.ScriptID, request.ScriptContentID, request.PolicyID, request.SetupExperienceScriptID, - priority, - request.SyncRequest, - isInternal, - request.UserID, ) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "new host script execution request") + return nil, ctxerr.Wrap(ctx, err, "new join script upcoming activity") } var script fleet.HostScriptResult - id, _ := result.LastInsertId() - err = sqlx.GetContext(ctx, tx, &script, getStmt, id) + err = sqlx.GetContext(ctx, tx, &script, getStmt, activityID) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting the created host script result to return") + return nil, ctxerr.Wrap(ctx, err, "getting the created host script activity to return") } return &script, nil } @@ -142,6 +156,7 @@ func truncateScriptResult(output string) string { func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *fleet.HostScriptResultPayload) (*fleet.HostScriptResult, string, error, ) { + // TODO(mna): this sets results of execution, so no impact on pending upcoming queue const resultExistsStmt = ` SELECT 1 @@ -263,6 +278,14 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { // pending host script executions are those without results in // host_script_results UNION those in the upcoming activities queue + // + // Note that: + // > Use of ORDER BY for individual SELECT statements implies nothing about + // > the order in which the rows appear in the final result because UNION by + // > default produces an unordered set of rows. + // + // So we need to order the final result set. + internalWhere := "" if onlyShowInternal { internalWhere = " AND is_internal = TRUE" @@ -273,39 +296,46 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID id, host_id, execution_id, - script_id + script_id, + 1 as topmost, + 0 as priority, + created_at FROM host_script_results WHERE host_id = ? AND %s - %s - ORDER BY - created_at ASC`, whereFilterPendingScript, internalWhere) + %s`, whereFilterPendingScript, internalWhere) if onlyShowInternal { internalWhere = " AND JSON_EXTRACT(payload, '$.is_internal') = 1" } listUAStmt := fmt.Sprintf(` SELECT - id, - host_id, - execution_id, - script_id + ua.id, + ua.host_id, + ua.execution_id, + sua.script_id, + 0 as topmost, + ua.priority, + ua.created_at FROM - upcoming_activities + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? AND - activity_type = 'script' AND + ua.host_id = ? AND + ua.activity_type = 'script' AND ( - JSON_EXTRACT(payload, '$.sync_request') = 0 OR - created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) + JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR + ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) ) - %s - ORDER BY - priority DESC, created_at ASC`, internalWhere) + %s`, internalWhere) - stmt := fmt.Sprintf(`(%s) UNION (%s)`, listHSRStmt, listUAStmt) + stmt := fmt.Sprintf(` + SELECT id, host_id, execution_id, script_id + FROM ( (%s) UNION (%s) ) as t + ORDER BY topmost DESC, priority DESC, created_at ASC`, listHSRStmt, listUAStmt) var results []*fleet.HostScriptResult seconds := int(constants.MaxServerWaitTime.Seconds()) @@ -329,11 +359,13 @@ func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, SELECT 1 FROM - upcoming_activities + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? AND - activity_type = 'script' AND - script_id = ? + ua.host_id = ? AND + ua.activity_type = 'script' AND + sua.script_id = ? ` var results []*uint @@ -348,6 +380,7 @@ func (ds *Datastore) GetHostScriptExecutionResult(ctx context.Context, execID st } func (ds *Datastore) getHostScriptExecutionResultDB(ctx context.Context, q sqlx.QueryerContext, execID string) (*fleet.HostScriptResult, error) { + // TODO(mna): this should probably return if it's pending still in upcoming_activities too const getStmt = ` SELECT hsr.id, @@ -537,6 +570,7 @@ var errDeleteScriptWithAssociatedPolicy = &fleet.ConflictError{Message: "Couldn' func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { return ds.withTx(ctx, func(tx sqlx.ExtContext) error { + // TODO(mna): delete pending execution from upcoming_activities _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE script_id = ? AND exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND)`, id, int(constants.MaxServerWaitTime.Seconds()), @@ -594,11 +628,15 @@ func (ds *Datastore) deletePendingHostScriptExecutionsForPolicy(ctx context.Cont deleteUAStmt := ` DELETE FROM + upcoming_activities + USING upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id WHERE - policy_id = ? AND - activity_type = 'script' AND - script_id IN ( + upcoming_activities.activity_type = 'script' AND + sua.policy_id = ? AND + sua.script_id IN ( SELECT id FROM scripts WHERE scripts.global_or_team_id = ? ) ` @@ -699,6 +737,8 @@ func (ds *Datastore) GetHostScriptDetails(ctx context.Context, hostID uint, team ExitCode *int64 `db:"exit_code"` } + // TODO(mna): must also look in upcoming queue, looks like this returns the latest + // execution/pending state for each script for a host? sql := ` SELECT s.id AS script_id, @@ -788,6 +828,7 @@ WHERE ` const unsetAllScriptsFromPolicies = `UPDATE policies SET script_id = NULL WHERE team_id = ?` + // TODO(mna): must clear pending executions from upcoming_activities too const clearAllPendingExecutions = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` @@ -805,6 +846,7 @@ WHERE name NOT IN (?) ` + // TODO(mna): must also clear pending executions from upcoming_activities const clearPendingExecutionsNotInList = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` @@ -820,6 +862,7 @@ ON DUPLICATE KEY UPDATE script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id) ` + // TODO(mna): must also clear pending executions from upcoming_activities const clearPendingExecutionsWithObsoleteScript = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id = ? AND script_content_id != ?` @@ -1371,6 +1414,7 @@ func updateHostLockWipeStatusFromResult(ctx context.Context, tx sqlx.ExtContext, } func updateUninstallStatusFromResult(ctx context.Context, tx sqlx.ExtContext, hostID uint, executionID string, exitCode int) error { + // TODO(mna): this sets results, no impact on pending upcoming queue stmt := ` UPDATE host_software_installs SET uninstall_script_exit_code = ? WHERE execution_id = ? AND host_id = ? ` @@ -1401,7 +1445,7 @@ WHERE SELECT 1 FROM setup_experience_scripts WHERE script_content_id = script_contents.id ) AND NOT EXISTS ( - SELECT 1 FROM upcoming_activities WHERE script_content_id = script_contents.id + SELECT 1 FROM script_upcoming_activities WHERE script_content_id = script_contents.id ) ` _, err := ds.writer(ctx).ExecContext(ctx, deleteStmt) diff --git a/server/datastore/mysql/scripts_test.go b/server/datastore/mysql/scripts_test.go index b3c83e08e8a4..c19ed4a35385 100644 --- a/server/datastore/mysql/scripts_test.go +++ b/server/datastore/mysql/scripts_test.go @@ -136,6 +136,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { */ // create another script execution request (null user id this time) + time.Sleep(time.Millisecond) // ensure a different timestamp createdScript2, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo2", @@ -199,6 +200,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { */ // create an async execution request + time.Sleep(time.Millisecond) // ensure a different timestamp createdScript3, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo 3", diff --git a/server/datastore/mysql/setup_experience.go b/server/datastore/mysql/setup_experience.go index a4ff177352b9..92425ba76182 100644 --- a/server/datastore/mysql/setup_experience.go +++ b/server/datastore/mysql/setup_experience.go @@ -305,6 +305,9 @@ func questionMarks(number int) string { } func (ds *Datastore) ListSetupExperienceResultsByHostUUID(ctx context.Context, hostUUID string) ([]*fleet.SetupExperienceStatusResult, error) { + // TODO(mna): this references the host software installs execution id, see if/how it + // impacts the upcoming queue (might be no impact if there's no FK, as the execution + // id is constant in the upcoming -> exec flow). const stmt = ` SELECT sesr.id, @@ -337,6 +340,7 @@ WHERE host_uuid = ? } func (ds *Datastore) UpdateSetupExperienceStatusResult(ctx context.Context, status *fleet.SetupExperienceStatusResult) error { + // TODO(mna): consider if this impacts upcoming queue const stmt = ` UPDATE setup_experience_status_results SET @@ -564,6 +568,7 @@ func (ds *Datastore) MaybeUpdateSetupExperienceVPPStatus(ctx context.Context, ho } func (ds *Datastore) MaybeUpdateSetupExperienceSoftwareInstallStatus(ctx context.Context, hostUUID string, executionID string, status fleet.SetupExperienceStatusResultStatus) (bool, error) { + // TODO(mna): consider if this impacts upcoming queue selectStmt := "SELECT id FROM setup_experience_status_results WHERE host_uuid = ? AND host_software_installs_execution_id = ?" updateStmt := "UPDATE setup_experience_status_results SET status = ? WHERE id = ?" diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 9d929c2289ec..2aed7f16ee6f 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -2216,6 +2216,10 @@ INNER JOIN software_cve scve ON scve.software_id = s.id softwareIsInstalledOnHostClause) } + // TODO(mna): this query is super complex, not even sure where upcoming activities fit in, but I think it does. + // Looks like it might impact upcoming software installs, scripts and VPP apps. May need to review the whole query + // to take a different approach, this is becoming unmaintainable. + // this statement lists only the software that is reported as installed on // the host or has been attempted to be installed on the host. stmtInstalled := fmt.Sprintf(` @@ -2311,7 +2315,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id -- on host (via installer or VPP app). If only available for install is -- requested, then the software installed on host clause is empty. ( %s hsi.host_id IS NOT NULL OR hvsi.host_id IS NOT NULL ) - AND + AND -- label membership check ( -- do the label membership check only for software installers @@ -2346,9 +2350,9 @@ INNER JOIN software_cve scve ON scve.software_id = s.id UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, @@ -2358,7 +2362,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = si.id @@ -2379,6 +2383,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id // attempted to be installed on the host, but that is available to be // installed on the host's platform. + // TODO(mna): I think this should exclude software and VPP apps that is pending in upcoming activities stmtAvailable := fmt.Sprintf(` SELECT st.id, @@ -2473,9 +2478,9 @@ INNER JOIN software_cve scve ON scve.software_id = s.id UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, @@ -2485,7 +2490,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = si.id @@ -2801,6 +2806,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id } func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { + // TODO(mna): this is to set the results, so it should not touch the upcoming queue const stmt = ` UPDATE host_software_installs @@ -2841,6 +2847,8 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f } func getInstalledByFleetSoftwareTitles(ctx context.Context, qc sqlx.QueryerContext, hostID uint) ([]fleet.SoftwareTitle, error) { + // TODO(mna): this only returns installed software, so no impact on upcoming queue + // We are overloading vpp_apps_count to indicate whether installed title is a VPP app or not. const stmt = ` SELECT @@ -2887,6 +2895,7 @@ WHERE hvsi.removed = 0 AND ncr.status = :mdm_status_acknowledged } func markHostSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(mna): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_software_installs hsi INNER JOIN software_installers si ON hsi.software_installer_id = si.id @@ -2905,6 +2914,7 @@ WHERE hsi.host_id = ? AND st.id IN (?) } func markHostVPPSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(mna): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_vpp_software_installs hvsi INNER JOIN vpp_apps vap ON hvsi.adam_id = vap.adam_id AND hvsi.platform = vap.platform diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index dd7a0aa091dc..d2bf8884476d 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -19,25 +19,43 @@ import ( func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { const stmt = ` - SELECT - execution_id - FROM - host_software_installs - WHERE - host_id = ? - AND - status = ? - ORDER BY - created_at ASC + SELECT + execution_id + FROM ( + SELECT + execution_id, + 1 as topmost, + 0 as priority, + created_at + FROM + host_software_installs + WHERE + host_id = ? AND + status = ? + UNION + SELECT + execution_id, + 0 as topmost, + priority, + created_at + FROM + upcoming_activities + WHERE + host_id = ? AND + activity_type = 'software_install' + ) as t + ORDER BY topmost DESC, priority ASC, created_at ASC ` var results []string - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID, fleet.SoftwareInstallPending); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, + stmt, hostID, fleet.SoftwareInstallPending, hostID); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending software installs") } return results, nil } func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { + // TODO(mna): must also look in upcoming queue const stmt = ` SELECT hsi.host_id AS host_id, @@ -456,6 +474,8 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } func (ds *Datastore) ValidateOrbitSoftwareInstallerAccess(ctx context.Context, hostID uint, installerID uint) (bool, error) { + // TODO(mna): this is ok to only look in host_software_installs, because orbit should not + // be able to get the installer until it is ready to install. query := ` SELECT 1 FROM @@ -671,6 +691,7 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, globalOrTeamID = *teamID } + // TODO(mna): must also delete from upcoming queue const deleteStmt = ` DELETE FROM host_software_installs @@ -691,13 +712,36 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { const ( - getInstallerStmt = `SELECT filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name - FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` - insertStmt = ` - INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` + getInstallerStmt = ` +SELECT + filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name +FROM + software_installers si + LEFT JOIN software_titles st + ON si.title_id = st.id +WHERE si.id = ?` + + //(execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'software_install', ?, + JSON_OBJECT( + 'self_service', ?, + 'installer_filename', ?, + 'version', ?, + 'software_title_name', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + ) + )` + + insertSIUAStmt = ` +INSERT INTO software_install_upcoming_activities + (upcoming_activity_id, software_installer_id, policy_id, software_title_id) +VALUES + (?, ?, ?, ?)` + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -727,24 +771,43 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui } var userID *uint + fleetInitiated := true if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { userID = &ctxUser.ID + fleetInitiated = false } - installID := uuid.NewString() - _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - installID, - hostID, - softwareInstallerID, - userID, - selfService, - policyID, - installerDetails.Filename, - installerDetails.Version, - installerDetails.TitleID, - installerDetails.TitleName, - ) + execID := uuid.NewString() + + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := ds.writer(ctx).ExecContext(ctx, insertUAStmt, + hostID, + 0, // TODO(mna): detect if this is software install for setup exp, to boost priority + userID, + fleetInitiated, + execID, + selfService, + installerDetails.Filename, + installerDetails.Version, + installerDetails.TitleName, + userID, + ) + if err != nil { + return err + } - return installID, ctxerr.Wrap(ctx, err, "inserting new install software request") + activityID, _ := res.LastInsertId() + _, err = ds.writer(ctx).ExecContext(ctx, insertSIUAStmt, + activityID, + softwareInstallerID, + policyID, + installerDetails.TitleID, + ) + if err != nil { + return err + } + return nil + }) + return execID, ctxerr.Wrap(ctx, err, "inserting new install software request") } func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { @@ -755,6 +818,9 @@ func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, inst func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Context, tx sqlx.ExtContext, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { if wasMetadataUpdated || wasPackageUpdated { // cancel pending installs/uninstalls + // TODO(mna): this deletes from host_script_results, but is actually related to software installs. + // Must add deletion from upcoming queue. + // TODO make this less naive; this assumes that installs/uninstalls execute and report back immediately _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE software_installer_id = ? AND status = 'pending_uninstall' @@ -782,6 +848,7 @@ func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Cont } func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { + // TODO(mna): must insert in upcoming queue const ( getInstallerStmt = `SELECT title_id, COALESCE(st.name, '[deleted title]') title_name FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` @@ -832,6 +899,7 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi } func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { + // TODO(mna): must also look in upcoming queue query := ` SELECT hsi.execution_id AS execution_id, @@ -880,6 +948,7 @@ WHERE func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { var dest fleet.SoftwareInstallerStatusSummary + // TODO(mna): must also look in upcoming queue for pending stmt := ` SELECT COALESCE(SUM( IF(status = :software_status_pending_install, 1, 0)), 0) AS pending_install, @@ -936,6 +1005,7 @@ func (ds *Datastore) vppAppJoin(appID fleet.VPPAppID, status fleet.SoftwareInsta default: // no change } + // TODO(mna): must join with upcoming queue for pending stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -985,6 +1055,7 @@ func (ds *Datastore) softwareInstallerJoin(installerID uint, status fleet.Softwa if status2 != "" { statusFilter = "hsi.status IN (:status, :status2)" } + // TODO(mna): must join with upcoming queue for pending stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -1012,6 +1083,8 @@ WHERE } func (ds *Datastore) GetHostLastInstallData(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { + // TODO(mna): I think that if there's none in host_software_installs, must take + // latest in upcoming queue. stmt := ` SELECT execution_id, hsi.status FROM host_software_installs hsi @@ -1088,6 +1161,8 @@ WHERE team_id = ? ` + // TODO(mna): this deletes from host_script_results but is related to software installs + // must add deletion from upcoming queue (here and many others below) const deleteAllPendingUninstallScriptExecutions = ` DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE status = 'pending_uninstall' @@ -1605,6 +1680,7 @@ func (ds *Datastore) HasSelfServiceSoftwareInstallers(ctx context.Context, hostP } func (ds *Datastore) GetSoftwareTitleNameFromExecutionID(ctx context.Context, executionID string) (string, error) { + // TODO(mna): must also look in upcoming queue stmt := ` SELECT name FROM software_titles st @@ -1709,24 +1785,24 @@ func (ds *Datastore) IsSoftwareInstallerLabelScoped(ctx context.Context, install UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, COUNT(lm.label_id) AS count_host_labels, - SUM(CASE - WHEN - lbl.created_at IS NOT NULL AND (SELECT label_updated_at FROM hosts WHERE id = :host_id) >= lbl.created_at THEN 1 - ELSE - 0 + SUM(CASE + WHEN + lbl.created_at IS NOT NULL AND (SELECT label_updated_at FROM hosts WHERE id = :host_id) >= lbl.created_at THEN 1 + ELSE + 0 END) as count_host_updated_after_labels FROM software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = :installer_id @@ -1756,7 +1832,7 @@ func (ds *Datastore) IsSoftwareInstallerLabelScoped(ctx context.Context, install return res, nil } -const labelScopedFilter = ` +const labelScopedFilter = ` SELECT 1 FROM ( diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 56d996ebe59e..d1fcc679d57b 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -129,77 +129,97 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall3, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall4, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host2.ID, - InstallUUID: hostInstall4, - InstallScriptExitCode: ptr.Int(0), - }) + pendingHost1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) require.NoError(t, err) + require.Equal(t, 2, len(pendingHost1)) + require.Equal(t, hostInstall1, pendingHost1[0]) + require.Equal(t, hostInstall2, pendingHost1[1]) - hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + pendingHost2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) require.NoError(t, err) + require.Equal(t, 2, len(pendingHost2)) + require.Equal(t, hostInstall3, pendingHost2[0]) + require.Equal(t, hostInstall4, pendingHost2[1]) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host2.ID, - InstallUUID: hostInstall5, - PreInstallConditionOutput: ptr.String(""), // pre-install query did not return results, so install failed - }) - require.NoError(t, err) + _ = installerID3 - installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) - require.NoError(t, err) - require.Equal(t, 2, len(installDetailsList1)) + // TODO(mna): uncomment the rest of this test once execution of upcoming activities is implemented + /* + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall4, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) - installDetailsList2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) - require.NoError(t, err) - require.Equal(t, 1, len(installDetailsList2)) + hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall5, + PreInstallConditionOutput: ptr.String(""), // pre-install query did not return results, so install failed + }) + require.NoError(t, err) - require.Contains(t, installDetailsList1, hostInstall1) - require.Contains(t, installDetailsList1, hostInstall2) + installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Equal(t, 2, len(installDetailsList1)) - require.Contains(t, installDetailsList2, hostInstall3) + installDetailsList2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) + require.NoError(t, err) + require.Equal(t, 1, len(installDetailsList2)) - exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) - require.NoError(t, err) + require.Contains(t, installDetailsList1, hostInstall1) + require.Contains(t, installDetailsList1, hostInstall2) - require.Equal(t, host1.ID, exec1.HostID) - require.Equal(t, hostInstall1, exec1.ExecutionID) - require.Equal(t, "hello DUCKY", exec1.InstallScript) - require.Equal(t, "world BIRD", exec1.PostInstallScript) - require.Equal(t, installerID1, exec1.InstallerID) - require.Equal(t, "SELECT 1", exec1.PreInstallCondition) - require.False(t, exec1.SelfService) - assert.Equal(t, "goodbye MONSTER", exec1.UninstallScript) + require.Contains(t, installDetailsList2, hostInstall3) - hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, true, nil) - require.NoError(t, err) + exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) + require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host1.ID, - InstallUUID: hostInstall6, - PreInstallConditionOutput: ptr.String("output"), - }) - require.NoError(t, err) + require.Equal(t, host1.ID, exec1.HostID) + require.Equal(t, hostInstall1, exec1.ExecutionID) + require.Equal(t, "hello DUCKY", exec1.InstallScript) + require.Equal(t, "world BIRD", exec1.PostInstallScript) + require.Equal(t, installerID1, exec1.InstallerID) + require.Equal(t, "SELECT 1", exec1.PreInstallCondition) + require.False(t, exec1.SelfService) + assert.Equal(t, "goodbye MONSTER", exec1.UninstallScript) - exec2, err := ds.GetSoftwareInstallDetails(ctx, hostInstall6) - require.NoError(t, err) + hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, true, nil) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host1.ID, + InstallUUID: hostInstall6, + PreInstallConditionOutput: ptr.String("output"), + }) + require.NoError(t, err) + + exec2, err := ds.GetSoftwareInstallDetails(ctx, hostInstall6) + require.NoError(t, err) - require.Equal(t, host1.ID, exec2.HostID) - require.Equal(t, hostInstall6, exec2.ExecutionID) - require.Equal(t, "banana", exec2.InstallScript) - require.Equal(t, "apple", exec2.PostInstallScript) - require.Equal(t, installerID3, exec2.InstallerID) - require.Equal(t, "SELECT 3", exec2.PreInstallCondition) - require.True(t, exec2.SelfService) + require.Equal(t, host1.ID, exec2.HostID) + require.Equal(t, hostInstall6, exec2.ExecutionID) + require.Equal(t, "banana", exec2.InstallScript) + require.Equal(t, "apple", exec2.PostInstallScript) + require.Equal(t, installerID3, exec2.InstallerID) + require.Equal(t, "SELECT 3", exec2.PreInstallCondition) + require.True(t, exec2.SelfService) + */ } func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index a98d8b13cab0..402cd0442885 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -60,6 +60,7 @@ func (ds *Datastore) GetSummaryHostVPPAppInstalls(ctx context.Context, teamID *u ) { var dest fleet.VPPAppStatusSummary + // TODO(mna): must consider upcoming queue for pending stmt := fmt.Sprintf(` SELECT COALESCE(SUM( IF(status = :software_status_pending, 1, 0)), 0) AS pending, @@ -500,6 +501,7 @@ WHERE vat.global_or_team_id = ? AND va.title_id = ? func (ds *Datastore) InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID, associatedEventID string, selfService bool, ) error { + // TODO(mna): insert in upcoming queue (and ensure command is not sent immediately) stmt := ` INSERT INTO host_vpp_software_installs (host_id, adam_id, platform, command_uuid, user_id, associated_event_id, self_service) @@ -525,6 +527,7 @@ func (ds *Datastore) GetPastActivityDataForVPPAppInstall(ctx context.Context, co return nil, nil, nil } + // TODO(mna): this should only ever be called for non-pending installs, no impact on upcoming queue stmt := ` SELECT u.name AS user_name,