Unified Queue: insert software installs in the upcoming queue and address more script changes

ee/server/service/setup_experience.go

@@ -178,6 +178,7 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string
 	case len(installersPending) > 0:
 		// enqueue installers
 		for _, installer := range installersPending {
+			// TODO(mna): this should be top priority as this is setup exp.
 			installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, false, nil)
 			if err != nil {
 				return false, ctxerr.Wrap(ctx, err, "queueing setup experience install request")

server/datastore/mysql/activities.go

@@ -248,6 +248,9 @@ func (ds *Datastore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [
 func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) {
 	// NOTE: Be sure to update both the count (here) and list statements (below)
 	// if the query condition is modified.
+
+	// TODO(mna): use the upcoming queue instead (but also include in-execution stuff
+	// that have no results yet)?
 	countStmts := []string{
 		`SELECT
 			COUNT(*) c
@@ -299,6 +302,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 
 	// NOTE: Be sure to update both the count (above) and list statements (below)
 	// if the query condition is modified.
+
+	// TODO(mna): use the upcoming queue instead (but also include in-execution stuff
+	// that have no results yet)?
 	listStmts := []string{
 		// list pending scripts
 		`SELECT
@@ -436,15 +442,15 @@ SELECT
 	) AS details
 FROM
 	host_vpp_software_installs hvsi
-INNER JOIN 
+INNER JOIN
 	nano_view_queue nvq ON nvq.command_uuid = hvsi.command_uuid
-LEFT OUTER JOIN 
+LEFT OUTER JOIN
 	users u ON hvsi.user_id = u.id
-LEFT OUTER JOIN 
+LEFT OUTER JOIN
 	host_display_names hdn ON hdn.host_id = hvsi.host_id
-LEFT OUTER JOIN 
+LEFT OUTER JOIN
 	vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform
-LEFT OUTER JOIN 
+LEFT OUTER JOIN
 	software_titles st ON st.id = vpa.title_id
 WHERE
 	nvq.status IS NULL

server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go

@@ -2,6 +2,7 @@ package tables
 
 import (
 	"database/sql"
+	"fmt"
 )
 
 func init() {
@@ -11,26 +12,52 @@ func init() {
 func Up_20250106162751(tx *sql.Tx) error {
 	_, err := tx.Exec(`
 CREATE TABLE upcoming_activities (
-	id             INT UNSIGNED NOT NULL AUTO_INCREMENT,
-	host_id        INT UNSIGNED NOT NULL,
+	id              BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+	host_id         INT UNSIGNED NOT NULL,
 
 	-- priority 0 is normal, > 0 is higher priority, < 0 is lower priority.
-	priority       INT NOT NULL DEFAULT 0,
+	priority        INT NOT NULL DEFAULT 0,
 
 	-- user_id is the user that triggered the activity, it may be null if the
 	-- activity is fleet-initiated or the user was deleted. Additional user
 	-- information (name, email, etc.) is stored in the JSON payload.
-	user_id        INT UNSIGNED NULL,
+	user_id         INT UNSIGNED NULL,
+	fleet_initiated TINYINT(1) NOT NULL DEFAULT 0,
 
 	-- type of activity to be executed, currently we only support those, but as
 	-- more activity types get added, we can enrich the ENUM with an ALTER TABLE.
-	activity_type  ENUM('script', 'software_install', 'vpp_app_install') NOT NULL,
+	activity_type  ENUM('script', 'software_install', 'software_uninstall', 'vpp_app_install') NOT NULL,
 
 	-- execution_id is the identifier of the activity that will be used when
 	-- executed - e.g. scripts and software installs have an execution_id, and
 	-- it is sometimes important to know it as soon as the activity is enqueued,
-	-- so we need to generate it immediately.
+	-- so we need to generate it immediately. Every activity will be identified
+	-- via this unique execution_id.
 	execution_id   VARCHAR(255) NOT NULL,
+	payload        JSON NOT NULL,
+
+	-- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues
+	created_at   DATETIME(6) NOT NULL DEFAULT NOW(6),
+	updated_at   DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6),
+
+	PRIMARY KEY (id),
+	UNIQUE KEY idx_upcoming_activities_execution_id (execution_id),
+	-- index for the common access pattern to get the next activity to execute
+	INDEX idx_upcoming_activities_host_id_priority_created_at (host_id, priority, created_at),
+	-- index for the common access pattern to get by activity type (e.g. deleting pending scripts)
+	INDEX idx_upcoming_activities_host_id_activity_type (activity_type, host_id),
+	CONSTRAINT fk_upcoming_activities_user_id
+		FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
+`,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create upcoming_activities: %w", err)
+	}
+
+	_, err = tx.Exec(`
+CREATE TABLE script_upcoming_activities (
+	upcoming_activity_id       BIGINT UNSIGNED NOT NULL,
 
 	-- those are all columns and not JSON fields because we need FKs on them to
 	-- do processing ON DELETE, otherwise we'd have to check for existence of
@@ -41,28 +68,60 @@ CREATE TABLE upcoming_activities (
 	policy_id                  INT UNSIGNED NULL,
 	setup_experience_script_id INT UNSIGNED NULL,
 
-	payload                    JSON NOT NULL,
-
 	-- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues
 	created_at   DATETIME(6) NOT NULL DEFAULT NOW(6),
 	updated_at   DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6),
 
-	PRIMARY KEY (id),
-	UNIQUE KEY idx_upcoming_activities_execution_id (execution_id),
-	INDEX idx_upcoming_activities_host_id_activity_type (host_id, priority, created_at, activity_type),
-	CONSTRAINT fk_upcoming_activities_script_id
+	PRIMARY KEY (upcoming_activity_id),
+	CONSTRAINT fk_script_upcoming_activities_upcoming_activity_id
+		FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE,
+	CONSTRAINT fk_script_upcoming_activities_script_id
 		FOREIGN KEY (script_id) REFERENCES scripts (id) ON DELETE SET NULL,
-	CONSTRAINT fk_upcoming_activities_script_content_id
+	CONSTRAINT fk_script_upcoming_activities_script_content_id
 		FOREIGN KEY (script_content_id) REFERENCES script_contents (id) ON DELETE CASCADE,
-	CONSTRAINT fk_upcoming_activities_policy_id
+	CONSTRAINT fk_script_upcoming_activities_policy_id
 		FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL,
-	CONSTRAINT fk_upcoming_activities_setup_experience_script_id
-		FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL,
-	CONSTRAINT fk_upcoming_activities_user_id
-		FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL
-) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`,
+	CONSTRAINT fk_script_upcoming_activities_setup_experience_script_id
+		FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
+`,
 	)
-	return err
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(`
+CREATE TABLE software_install_upcoming_activities (
+	upcoming_activity_id       BIGINT UNSIGNED NOT NULL,
+
+	-- those are all columns and not JSON fields because we need FKs on them to
+	-- do processing ON DELETE, otherwise we'd have to check for existence of
+	-- each one when executing the activity (we need the enqueue next activity
+	-- action to be efficient).
+	software_installer_id      INT UNSIGNED NULL,
+	policy_id                  INT UNSIGNED NULL,
+	software_title_id          INT UNSIGNED NULL,
+
+	-- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues
+	created_at   DATETIME(6) NOT NULL DEFAULT NOW(6),
+	updated_at   DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6),
+
+	PRIMARY KEY (upcoming_activity_id),
+	CONSTRAINT fk_software_install_upcoming_activities_upcoming_activity_id
+		FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE,
+	CONSTRAINT fk_software_install_upcoming_activities_software_installer_id
+		FOREIGN KEY (software_installer_id) REFERENCES software_installers (id) ON DELETE SET NULL ON UPDATE CASCADE,
+	CONSTRAINT fk_software_install_upcoming_activities_policy_id
+		FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL,
+	CONSTRAINT fk_software_install_upcoming_activities_software_title_id
+		FOREIGN KEY (software_title_id) REFERENCES software_titles (id) ON DELETE SET NULL ON UPDATE CASCADE
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
+`,
+	)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 func Down_20250106162751(tx *sql.Tx) error {

server/datastore/mysql/schema.sql

@@ -1644,6 +1644,28 @@ CREATE TABLE `script_contents` (
 /*!40101 SET character_set_client = @saved_cs_client */;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `script_upcoming_activities` (
+  `upcoming_activity_id` bigint unsigned NOT NULL,
+  `script_id` int unsigned DEFAULT NULL,
+  `script_content_id` int unsigned DEFAULT NULL,
+  `policy_id` int unsigned DEFAULT NULL,
+  `setup_experience_script_id` int unsigned DEFAULT NULL,
+  `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+  `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+  PRIMARY KEY (`upcoming_activity_id`),
+  KEY `fk_script_upcoming_activities_script_id` (`script_id`),
+  KEY `fk_script_upcoming_activities_script_content_id` (`script_content_id`),
+  KEY `fk_script_upcoming_activities_policy_id` (`policy_id`),
+  KEY `fk_script_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`),
+  CONSTRAINT `fk_script_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL,
+  CONSTRAINT `fk_script_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE,
+  CONSTRAINT `fk_script_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL,
+  CONSTRAINT `fk_script_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL,
+  CONSTRAINT `fk_script_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `scripts` (
   `id` int unsigned NOT NULL AUTO_INCREMENT,
   `team_id` int unsigned DEFAULT NULL,
@@ -1797,6 +1819,25 @@ CREATE TABLE `software_host_counts` (
 /*!40101 SET character_set_client = @saved_cs_client */;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `software_install_upcoming_activities` (
+  `upcoming_activity_id` bigint unsigned NOT NULL,
+  `software_installer_id` int unsigned DEFAULT NULL,
+  `policy_id` int unsigned DEFAULT NULL,
+  `software_title_id` int unsigned DEFAULT NULL,
+  `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+  `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+  PRIMARY KEY (`upcoming_activity_id`),
+  KEY `fk_software_install_upcoming_activities_software_installer_id` (`software_installer_id`),
+  KEY `fk_software_install_upcoming_activities_policy_id` (`policy_id`),
+  KEY `fk_software_install_upcoming_activities_software_title_id` (`software_title_id`),
+  CONSTRAINT `fk_software_install_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL,
+  CONSTRAINT `fk_software_install_upcoming_activities_software_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+  CONSTRAINT `fk_software_install_upcoming_activities_software_title_id` FOREIGN KEY (`software_title_id`) REFERENCES `software_titles` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+  CONSTRAINT `fk_software_install_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `software_installer_labels` (
   `id` int unsigned NOT NULL AUTO_INCREMENT,
   `software_installer_id` int unsigned NOT NULL,
@@ -1912,31 +1953,21 @@ CREATE TABLE `teams` (
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `upcoming_activities` (
-  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `id` bigint unsigned NOT NULL AUTO_INCREMENT,
   `host_id` int unsigned NOT NULL,
   `priority` int NOT NULL DEFAULT '0',
   `user_id` int unsigned DEFAULT NULL,
-  `activity_type` enum('script','software_install','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL,
+  `fleet_initiated` tinyint(1) NOT NULL DEFAULT '0',
+  `activity_type` enum('script','software_install','software_uninstall','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL,
   `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
-  `script_id` int unsigned DEFAULT NULL,
-  `script_content_id` int unsigned DEFAULT NULL,
-  `policy_id` int unsigned DEFAULT NULL,
-  `setup_experience_script_id` int unsigned DEFAULT NULL,
   `payload` json NOT NULL,
   `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
   `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
   PRIMARY KEY (`id`),
   UNIQUE KEY `idx_upcoming_activities_execution_id` (`execution_id`),
-  KEY `idx_upcoming_activities_host_id_activity_type` (`host_id`,`priority`,`created_at`,`activity_type`),
-  KEY `fk_upcoming_activities_script_id` (`script_id`),
-  KEY `fk_upcoming_activities_script_content_id` (`script_content_id`),
-  KEY `fk_upcoming_activities_policy_id` (`policy_id`),
-  KEY `fk_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`),
+  KEY `idx_upcoming_activities_host_id_priority_created_at` (`host_id`,`priority`,`created_at`),
+  KEY `idx_upcoming_activities_host_id_activity_type` (`activity_type`,`host_id`),
   KEY `fk_upcoming_activities_user_id` (`user_id`),
-  CONSTRAINT `fk_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL,
-  CONSTRAINT `fk_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE,
-  CONSTRAINT `fk_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL,
-  CONSTRAINT `fk_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL,
   CONSTRAINT `fk_upcoming_activities_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;