From 28a72ac47f1086f744fe4edcd187efc1943712bb Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 08:31:25 -0500 Subject: [PATCH 01/15] Switch upcoming activities id to bigint --- .../migrations/tables/20250106162751_AddUnifiedQueueTable.go | 2 +- server/datastore/mysql/schema.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go index 8f5c3e73542e..9f2877250d4e 100644 --- a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go @@ -11,7 +11,7 @@ func init() { func Up_20250106162751(tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE upcoming_activities ( - id INT UNSIGNED NOT NULL AUTO_INCREMENT, + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, host_id INT UNSIGNED NOT NULL, -- priority 0 is normal, > 0 is higher priority, < 0 is lower priority. diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 3e92aef3f579..eaf0b0b5d57b 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1912,7 +1912,7 @@ CREATE TABLE `teams` ( /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `upcoming_activities` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, + `id` bigint unsigned NOT NULL AUTO_INCREMENT, `host_id` int unsigned NOT NULL, `priority` int NOT NULL DEFAULT '0', `user_id` int unsigned DEFAULT NULL, From debf5750bead244fad77859e1853d4aff51b082a Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 10:56:36 -0500 Subject: [PATCH 02/15] Mark all places where SQL statements possibly need to change --- server/datastore/mysql/activities.go | 16 +++++--- server/datastore/mysql/scripts.go | 9 +++++ server/datastore/mysql/setup_experience.go | 5 +++ server/datastore/mysql/software.go | 28 ++++++++----- server/datastore/mysql/software_installers.go | 39 ++++++++++++++----- server/datastore/mysql/vpp.go | 3 ++ 6 files changed, 76 insertions(+), 24 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index bfe732426e87..99b342d283c8 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -248,6 +248,9 @@ func (ds *Datastore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { // NOTE: Be sure to update both the count (here) and list statements (below) // if the query condition is modified. + + // TODO(mna): use the upcoming queue instead (but also include in-execution stuff + // that have no results yet)? countStmts := []string{ `SELECT COUNT(*) c @@ -299,6 +302,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint // NOTE: Be sure to update both the count (above) and list statements (below) // if the query condition is modified. + + // TODO(mna): use the upcoming queue instead (but also include in-execution stuff + // that have no results yet)? listStmts := []string{ // list pending scripts `SELECT @@ -436,15 +442,15 @@ SELECT ) AS details FROM host_vpp_software_installs hvsi -INNER JOIN +INNER JOIN nano_view_queue nvq ON nvq.command_uuid = hvsi.command_uuid -LEFT OUTER JOIN +LEFT OUTER JOIN users u ON hvsi.user_id = u.id -LEFT OUTER JOIN +LEFT OUTER JOIN host_display_names hdn ON hdn.host_id = hvsi.host_id -LEFT OUTER JOIN +LEFT OUTER JOIN vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform -LEFT OUTER JOIN +LEFT OUTER JOIN software_titles st ON st.id = vpa.title_id WHERE nvq.status IS NULL diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 6118f3786675..3ad275e473d7 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -142,6 +142,7 @@ func truncateScriptResult(output string) string { func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *fleet.HostScriptResultPayload) (*fleet.HostScriptResult, string, error, ) { + // TODO(mna): this sets results of execution, so no impact on pending upcoming queue const resultExistsStmt = ` SELECT 1 @@ -348,6 +349,7 @@ func (ds *Datastore) GetHostScriptExecutionResult(ctx context.Context, execID st } func (ds *Datastore) getHostScriptExecutionResultDB(ctx context.Context, q sqlx.QueryerContext, execID string) (*fleet.HostScriptResult, error) { + // TODO(mna): this should probably return if it's pending still in upcoming_activities too const getStmt = ` SELECT hsr.id, @@ -537,6 +539,7 @@ var errDeleteScriptWithAssociatedPolicy = &fleet.ConflictError{Message: "Couldn' func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { return ds.withTx(ctx, func(tx sqlx.ExtContext) error { + // TODO(mna): delete pending execution from upcoming_activities _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE script_id = ? AND exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND)`, id, int(constants.MaxServerWaitTime.Seconds()), @@ -699,6 +702,8 @@ func (ds *Datastore) GetHostScriptDetails(ctx context.Context, hostID uint, team ExitCode *int64 `db:"exit_code"` } + // TODO(mna): must also look in upcoming queue, looks like this returns the latest + // execution/pending state for each script for a host? sql := ` SELECT s.id AS script_id, @@ -788,6 +793,7 @@ WHERE ` const unsetAllScriptsFromPolicies = `UPDATE policies SET script_id = NULL WHERE team_id = ?` + // TODO(mna): must clear pending executions from upcoming_activities too const clearAllPendingExecutions = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` @@ -805,6 +811,7 @@ WHERE name NOT IN (?) ` + // TODO(mna): must also clear pending executions from upcoming_activities const clearPendingExecutionsNotInList = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` @@ -820,6 +827,7 @@ ON DUPLICATE KEY UPDATE script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id) ` + // TODO(mna): must also clear pending executions from upcoming_activities const clearPendingExecutionsWithObsoleteScript = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) AND script_id = ? AND script_content_id != ?` @@ -1371,6 +1379,7 @@ func updateHostLockWipeStatusFromResult(ctx context.Context, tx sqlx.ExtContext, } func updateUninstallStatusFromResult(ctx context.Context, tx sqlx.ExtContext, hostID uint, executionID string, exitCode int) error { + // TODO(mna): this sets results, no impact on pending upcoming queue stmt := ` UPDATE host_software_installs SET uninstall_script_exit_code = ? WHERE execution_id = ? AND host_id = ? ` diff --git a/server/datastore/mysql/setup_experience.go b/server/datastore/mysql/setup_experience.go index a4ff177352b9..92425ba76182 100644 --- a/server/datastore/mysql/setup_experience.go +++ b/server/datastore/mysql/setup_experience.go @@ -305,6 +305,9 @@ func questionMarks(number int) string { } func (ds *Datastore) ListSetupExperienceResultsByHostUUID(ctx context.Context, hostUUID string) ([]*fleet.SetupExperienceStatusResult, error) { + // TODO(mna): this references the host software installs execution id, see if/how it + // impacts the upcoming queue (might be no impact if there's no FK, as the execution + // id is constant in the upcoming -> exec flow). const stmt = ` SELECT sesr.id, @@ -337,6 +340,7 @@ WHERE host_uuid = ? } func (ds *Datastore) UpdateSetupExperienceStatusResult(ctx context.Context, status *fleet.SetupExperienceStatusResult) error { + // TODO(mna): consider if this impacts upcoming queue const stmt = ` UPDATE setup_experience_status_results SET @@ -564,6 +568,7 @@ func (ds *Datastore) MaybeUpdateSetupExperienceVPPStatus(ctx context.Context, ho } func (ds *Datastore) MaybeUpdateSetupExperienceSoftwareInstallStatus(ctx context.Context, hostUUID string, executionID string, status fleet.SetupExperienceStatusResultStatus) (bool, error) { + // TODO(mna): consider if this impacts upcoming queue selectStmt := "SELECT id FROM setup_experience_status_results WHERE host_uuid = ? AND host_software_installs_execution_id = ?" updateStmt := "UPDATE setup_experience_status_results SET status = ? WHERE id = ?" diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 9d929c2289ec..2aed7f16ee6f 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -2216,6 +2216,10 @@ INNER JOIN software_cve scve ON scve.software_id = s.id softwareIsInstalledOnHostClause) } + // TODO(mna): this query is super complex, not even sure where upcoming activities fit in, but I think it does. + // Looks like it might impact upcoming software installs, scripts and VPP apps. May need to review the whole query + // to take a different approach, this is becoming unmaintainable. + // this statement lists only the software that is reported as installed on // the host or has been attempted to be installed on the host. stmtInstalled := fmt.Sprintf(` @@ -2311,7 +2315,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id -- on host (via installer or VPP app). If only available for install is -- requested, then the software installed on host clause is empty. ( %s hsi.host_id IS NOT NULL OR hvsi.host_id IS NOT NULL ) - AND + AND -- label membership check ( -- do the label membership check only for software installers @@ -2346,9 +2350,9 @@ INNER JOIN software_cve scve ON scve.software_id = s.id UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, @@ -2358,7 +2362,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = si.id @@ -2379,6 +2383,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id // attempted to be installed on the host, but that is available to be // installed on the host's platform. + // TODO(mna): I think this should exclude software and VPP apps that is pending in upcoming activities stmtAvailable := fmt.Sprintf(` SELECT st.id, @@ -2473,9 +2478,9 @@ INNER JOIN software_cve scve ON scve.software_id = s.id UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, @@ -2485,7 +2490,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = si.id @@ -2801,6 +2806,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id } func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { + // TODO(mna): this is to set the results, so it should not touch the upcoming queue const stmt = ` UPDATE host_software_installs @@ -2841,6 +2847,8 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f } func getInstalledByFleetSoftwareTitles(ctx context.Context, qc sqlx.QueryerContext, hostID uint) ([]fleet.SoftwareTitle, error) { + // TODO(mna): this only returns installed software, so no impact on upcoming queue + // We are overloading vpp_apps_count to indicate whether installed title is a VPP app or not. const stmt = ` SELECT @@ -2887,6 +2895,7 @@ WHERE hvsi.removed = 0 AND ncr.status = :mdm_status_acknowledged } func markHostSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(mna): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_software_installs hsi INNER JOIN software_installers si ON hsi.software_installer_id = si.id @@ -2905,6 +2914,7 @@ WHERE hsi.host_id = ? AND st.id IN (?) } func markHostVPPSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(mna): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_vpp_software_installs hvsi INNER JOIN vpp_apps vap ON hvsi.adam_id = vap.adam_id AND hvsi.platform = vap.platform diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index dd7a0aa091dc..0403d88bdcff 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -18,6 +18,7 @@ import ( ) func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { + // TODO(mna): must union with upcoming queue const stmt = ` SELECT execution_id @@ -38,6 +39,7 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin } func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { + // TODO(mna): must also look in upcoming queue const stmt = ` SELECT hsi.host_id AS host_id, @@ -456,6 +458,8 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } func (ds *Datastore) ValidateOrbitSoftwareInstallerAccess(ctx context.Context, hostID uint, installerID uint) (bool, error) { + // TODO(mna): this is ok to only look in host_software_installs, because orbit should not + // be able to get the installer until it is ready to install. query := ` SELECT 1 FROM @@ -671,6 +675,7 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, globalOrTeamID = *teamID } + // TODO(mna): must also delete from upcoming queue const deleteStmt = ` DELETE FROM host_software_installs @@ -690,6 +695,7 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, } func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { + // TODO(mna): must insert in upcoming queue const ( getInstallerStmt = `SELECT filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` @@ -755,6 +761,9 @@ func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, inst func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Context, tx sqlx.ExtContext, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { if wasMetadataUpdated || wasPackageUpdated { // cancel pending installs/uninstalls + // TODO(mna): this deletes from host_script_results, but is actually related to software installs. + // Must add deletion from upcoming queue. + // TODO make this less naive; this assumes that installs/uninstalls execute and report back immediately _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE software_installer_id = ? AND status = 'pending_uninstall' @@ -782,6 +791,7 @@ func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Cont } func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { + // TODO(mna): must insert in upcoming queue const ( getInstallerStmt = `SELECT title_id, COALESCE(st.name, '[deleted title]') title_name FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` @@ -832,6 +842,7 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi } func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { + // TODO(mna): must also look in upcoming queue query := ` SELECT hsi.execution_id AS execution_id, @@ -880,6 +891,7 @@ WHERE func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { var dest fleet.SoftwareInstallerStatusSummary + // TODO(mna): must also look in upcoming queue for pending stmt := ` SELECT COALESCE(SUM( IF(status = :software_status_pending_install, 1, 0)), 0) AS pending_install, @@ -936,6 +948,7 @@ func (ds *Datastore) vppAppJoin(appID fleet.VPPAppID, status fleet.SoftwareInsta default: // no change } + // TODO(mna): must join with upcoming queue for pending stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -985,6 +998,7 @@ func (ds *Datastore) softwareInstallerJoin(installerID uint, status fleet.Softwa if status2 != "" { statusFilter = "hsi.status IN (:status, :status2)" } + // TODO(mna): must join with upcoming queue for pending stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -1012,6 +1026,8 @@ WHERE } func (ds *Datastore) GetHostLastInstallData(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { + // TODO(mna): I think that if there's none in host_software_installs, must take + // latest in upcoming queue. stmt := ` SELECT execution_id, hsi.status FROM host_software_installs hsi @@ -1088,6 +1104,8 @@ WHERE team_id = ? ` + // TODO(mna): this deletes from host_script_results but is related to software installs + // must add deletion from upcoming queue (here and many others below) const deleteAllPendingUninstallScriptExecutions = ` DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE status = 'pending_uninstall' @@ -1605,6 +1623,7 @@ func (ds *Datastore) HasSelfServiceSoftwareInstallers(ctx context.Context, hostP } func (ds *Datastore) GetSoftwareTitleNameFromExecutionID(ctx context.Context, executionID string) (string, error) { + // TODO(mna): must also look in upcoming queue stmt := ` SELECT name FROM software_titles st @@ -1709,24 +1728,24 @@ func (ds *Datastore) IsSoftwareInstallerLabelScoped(ctx context.Context, install UNION - -- exclude any, ignore software that depends on labels created - -- _after_ the label_updated_at timestamp of the host (because - -- we don't have results for that label yet, the host may or may + -- exclude any, ignore software that depends on labels created + -- _after_ the label_updated_at timestamp of the host (because + -- we don't have results for that label yet, the host may or may -- not be a member). SELECT COUNT(*) AS count_installer_labels, COUNT(lm.label_id) AS count_host_labels, - SUM(CASE - WHEN - lbl.created_at IS NOT NULL AND (SELECT label_updated_at FROM hosts WHERE id = :host_id) >= lbl.created_at THEN 1 - ELSE - 0 + SUM(CASE + WHEN + lbl.created_at IS NOT NULL AND (SELECT label_updated_at FROM hosts WHERE id = :host_id) >= lbl.created_at THEN 1 + ELSE + 0 END) as count_host_updated_after_labels FROM software_installer_labels sil LEFT OUTER JOIN labels lbl ON lbl.id = sil.label_id - LEFT OUTER JOIN label_membership lm + LEFT OUTER JOIN label_membership lm ON lm.label_id = sil.label_id AND lm.host_id = :host_id WHERE sil.software_installer_id = :installer_id @@ -1756,7 +1775,7 @@ func (ds *Datastore) IsSoftwareInstallerLabelScoped(ctx context.Context, install return res, nil } -const labelScopedFilter = ` +const labelScopedFilter = ` SELECT 1 FROM ( diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index a98d8b13cab0..402cd0442885 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -60,6 +60,7 @@ func (ds *Datastore) GetSummaryHostVPPAppInstalls(ctx context.Context, teamID *u ) { var dest fleet.VPPAppStatusSummary + // TODO(mna): must consider upcoming queue for pending stmt := fmt.Sprintf(` SELECT COALESCE(SUM( IF(status = :software_status_pending, 1, 0)), 0) AS pending, @@ -500,6 +501,7 @@ WHERE vat.global_or_team_id = ? AND va.title_id = ? func (ds *Datastore) InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID, associatedEventID string, selfService bool, ) error { + // TODO(mna): insert in upcoming queue (and ensure command is not sent immediately) stmt := ` INSERT INTO host_vpp_software_installs (host_id, adam_id, platform, command_uuid, user_id, associated_event_id, self_service) @@ -525,6 +527,7 @@ func (ds *Datastore) GetPastActivityDataForVPPAppInstall(ctx context.Context, co return nil, nil, nil } + // TODO(mna): this should only ever be called for non-pending installs, no impact on upcoming queue stmt := ` SELECT u.name AS user_name, From 8a8e4fa8790dd2dd10a9bea399bc82155da9c4c7 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 11:47:58 -0500 Subject: [PATCH 03/15] Use a join table for activity-type-specific columns --- .../20250106162751_AddUnifiedQueueTable.go | 62 +++++++++++++------ server/datastore/mysql/schema.sql | 35 +++++++---- 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go index 9f2877250d4e..28ef1dd7e485 100644 --- a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go @@ -2,6 +2,7 @@ package tables import ( "database/sql" + "fmt" ) func init() { @@ -11,16 +12,17 @@ func init() { func Up_20250106162751(tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE upcoming_activities ( - id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, - host_id INT UNSIGNED NOT NULL, + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + host_id INT UNSIGNED NOT NULL, -- priority 0 is normal, > 0 is higher priority, < 0 is lower priority. - priority INT NOT NULL DEFAULT 0, + priority INT NOT NULL DEFAULT 0, -- user_id is the user that triggered the activity, it may be null if the -- activity is fleet-initiated or the user was deleted. Additional user -- information (name, email, etc.) is stored in the JSON payload. - user_id INT UNSIGNED NULL, + user_id INT UNSIGNED NULL, + fleet_initiated TINYINT(1) NOT NULL DEFAULT 0, -- type of activity to be executed, currently we only support those, but as -- more activity types get added, we can enrich the ENUM with an ALTER TABLE. @@ -29,8 +31,30 @@ CREATE TABLE upcoming_activities ( -- execution_id is the identifier of the activity that will be used when -- executed - e.g. scripts and software installs have an execution_id, and -- it is sometimes important to know it as soon as the activity is enqueued, - -- so we need to generate it immediately. + -- so we need to generate it immediately. Every activity will be identified + -- via this unique execution_id. execution_id VARCHAR(255) NOT NULL, + payload JSON NOT NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (id), + UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), + INDEX idx_upcoming_activities_host_id_activity_type (host_id, priority, created_at, activity_type), + CONSTRAINT fk_upcoming_activities_user_id + FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return fmt.Errorf("failed to create upcoming_activities: %w", err) + } + + _, err = tx.Exec(` +CREATE TABLE script_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, -- those are all columns and not JSON fields because we need FKs on them to -- do processing ON DELETE, otherwise we'd have to check for existence of @@ -41,28 +65,28 @@ CREATE TABLE upcoming_activities ( policy_id INT UNSIGNED NULL, setup_experience_script_id INT UNSIGNED NULL, - payload JSON NOT NULL, - -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues created_at DATETIME(6) NOT NULL DEFAULT NOW(6), updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), - PRIMARY KEY (id), - UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), - INDEX idx_upcoming_activities_host_id_activity_type (host_id, priority, created_at, activity_type), - CONSTRAINT fk_upcoming_activities_script_id + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_script_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_script_upcoming_activities_script_id FOREIGN KEY (script_id) REFERENCES scripts (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_script_content_id + CONSTRAINT fk_script_upcoming_activities_script_content_id FOREIGN KEY (script_content_id) REFERENCES script_contents (id) ON DELETE CASCADE, - CONSTRAINT fk_upcoming_activities_policy_id + CONSTRAINT fk_script_upcoming_activities_policy_id FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_setup_experience_script_id - FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL, - CONSTRAINT fk_upcoming_activities_user_id - FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL -) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`, + CONSTRAINT fk_script_upcoming_activities_setup_experience_script_id + FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, ) - return err + if err != nil { + return err + } + return nil } func Down_20250106162751(tx *sql.Tx) error { diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index eaf0b0b5d57b..e67983223ee3 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1644,6 +1644,28 @@ CREATE TABLE `script_contents` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `script_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `script_id` int unsigned DEFAULT NULL, + `script_content_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `setup_experience_script_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_script_upcoming_activities_script_id` (`script_id`), + KEY `fk_script_upcoming_activities_script_content_id` (`script_content_id`), + KEY `fk_script_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_script_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`), + CONSTRAINT `fk_script_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_script_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `scripts` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `team_id` int unsigned DEFAULT NULL, @@ -1916,27 +1938,16 @@ CREATE TABLE `upcoming_activities` ( `host_id` int unsigned NOT NULL, `priority` int NOT NULL DEFAULT '0', `user_id` int unsigned DEFAULT NULL, + `fleet_initiated` tinyint(1) NOT NULL DEFAULT '0', `activity_type` enum('script','software_install','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - `script_id` int unsigned DEFAULT NULL, - `script_content_id` int unsigned DEFAULT NULL, - `policy_id` int unsigned DEFAULT NULL, - `setup_experience_script_id` int unsigned DEFAULT NULL, `payload` json NOT NULL, `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`id`), UNIQUE KEY `idx_upcoming_activities_execution_id` (`execution_id`), KEY `idx_upcoming_activities_host_id_activity_type` (`host_id`,`priority`,`created_at`,`activity_type`), - KEY `fk_upcoming_activities_script_id` (`script_id`), - KEY `fk_upcoming_activities_script_content_id` (`script_content_id`), - KEY `fk_upcoming_activities_policy_id` (`policy_id`), - KEY `fk_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`), KEY `fk_upcoming_activities_user_id` (`user_id`), - CONSTRAINT `fk_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, - CONSTRAINT `fk_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL, - CONSTRAINT `fk_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL, CONSTRAINT `fk_upcoming_activities_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; From dbe853263b7e93c1a3ea4020c782fc7e4ec220dc Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 14:05:26 -0500 Subject: [PATCH 04/15] Update insertion/deletion with new join table --- server/datastore/mysql/scripts.go | 94 +++++++++++++++++++------------ 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 3ad275e473d7..3cb454f25891 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -62,14 +62,11 @@ func (ds *Datastore) NewInternalScriptExecutionRequest(ctx context.Context, requ func newHostScriptExecutionRequest(ctx context.Context, tx sqlx.ExtContext, request *fleet.HostScriptRequestPayload, isInternal bool) (*fleet.HostScriptResult, error) { const ( - insStmt = ` + insUAStmt = ` INSERT INTO upcoming_activities - ( - host_id, user_id, activity_type, execution_id, script_id, script_content_id, - policy_id, setup_experience_script_id, priority, payload - ) + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) VALUES - (?, ?, 'script', ?, ?, ?, ?, ?, ?, + (?, ?, ?, ?, 'script', ?, JSON_OBJECT( 'sync_request', ?, 'is_internal', ?, @@ -77,15 +74,24 @@ VALUES ) )` + insSUAStmt = ` +INSERT INTO script_upcoming_activities + (upcoming_activity_id, script_id, script_content_id, policy_id, setup_experience_script_id) +VALUES + (?, ?, ?, ?, ?) +` + getStmt = ` SELECT - ua.id, ua.host_id, ua.execution_id, ua.created_at, ua.script_id, ua.policy_id, ua.user_id, + ua.id, ua.host_id, ua.execution_id, ua.created_at, sua.script_id, sua.policy_id, ua.user_id, JSON_EXTRACT(payload, '$.sync_request') AS sync_request, - sc.contents as script_contents, ua.setup_experience_script_id + sc.contents as script_contents, sua.setup_experience_script_id FROM upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id INNER JOIN script_contents sc - ON ua.script_content_id = sc.id + ON sua.script_content_id = sc.id WHERE ua.id = ? ` @@ -99,28 +105,36 @@ WHERE } execID := uuid.New().String() - result, err := tx.ExecContext(ctx, insStmt, + result, err := tx.ExecContext(ctx, insUAStmt, request.HostID, + priority, request.UserID, + false, // TODO(mna): do we have fleet-initiated scripts? execID, + request.SyncRequest, + isInternal, + request.UserID, + ) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "new script upcoming activity") + } + + activityID, _ := result.LastInsertId() + _, err = tx.ExecContext(ctx, insSUAStmt, + activityID, request.ScriptID, request.ScriptContentID, request.PolicyID, request.SetupExperienceScriptID, - priority, - request.SyncRequest, - isInternal, - request.UserID, ) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "new host script execution request") + return nil, ctxerr.Wrap(ctx, err, "new join script upcoming activity") } var script fleet.HostScriptResult - id, _ := result.LastInsertId() - err = sqlx.GetContext(ctx, tx, &script, getStmt, id) + err = sqlx.GetContext(ctx, tx, &script, getStmt, activityID) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting the created host script result to return") + return nil, ctxerr.Wrap(ctx, err, "getting the created host script activity to return") } return &script, nil } @@ -289,22 +303,24 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID } listUAStmt := fmt.Sprintf(` SELECT - id, - host_id, - execution_id, - script_id + ua.id, + ua.host_id, + ua.execution_id, + sua.script_id FROM - upcoming_activities + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? AND - activity_type = 'script' AND + ua.host_id = ? AND + ua.activity_type = 'script' AND ( - JSON_EXTRACT(payload, '$.sync_request') = 0 OR - created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) + JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR + ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) ) %s ORDER BY - priority DESC, created_at ASC`, internalWhere) + ua.priority DESC, ua.created_at ASC`, internalWhere) stmt := fmt.Sprintf(`(%s) UNION (%s)`, listHSRStmt, listUAStmt) @@ -330,11 +346,13 @@ func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, SELECT 1 FROM - upcoming_activities + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? AND - activity_type = 'script' AND - script_id = ? + ua.host_id = ? AND + ua.activity_type = 'script' AND + sua.script_id = ? ` var results []*uint @@ -597,11 +615,15 @@ func (ds *Datastore) deletePendingHostScriptExecutionsForPolicy(ctx context.Cont deleteUAStmt := ` DELETE FROM + upcoming_activities + USING upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id WHERE - policy_id = ? AND - activity_type = 'script' AND - script_id IN ( + upcoming_activities.activity_type = 'script' AND + sua.policy_id = ? AND + sua.script_id IN ( SELECT id FROM scripts WHERE scripts.global_or_team_id = ? ) ` @@ -1410,7 +1432,7 @@ WHERE SELECT 1 FROM setup_experience_scripts WHERE script_content_id = script_contents.id ) AND NOT EXISTS ( - SELECT 1 FROM upcoming_activities WHERE script_content_id = script_contents.id + SELECT 1 FROM script_upcoming_activities WHERE script_content_id = script_contents.id ) ` _, err := ds.writer(ctx).ExecContext(ctx, deleteStmt) From e02c363dc35a06fa32780d66c19094e242e0cab9 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 15:02:29 -0500 Subject: [PATCH 05/15] Add join table for software installs/uninstalls --- .../20250106162751_AddUnifiedQueueTable.go | 39 ++++++++++++++++++- server/datastore/mysql/schema.sql | 24 +++++++++++- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go index 28ef1dd7e485..1100b8196500 100644 --- a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go @@ -26,7 +26,7 @@ CREATE TABLE upcoming_activities ( -- type of activity to be executed, currently we only support those, but as -- more activity types get added, we can enrich the ENUM with an ALTER TABLE. - activity_type ENUM('script', 'software_install', 'vpp_app_install') NOT NULL, + activity_type ENUM('script', 'software_install', 'software_uninstall', 'vpp_app_install') NOT NULL, -- execution_id is the identifier of the activity that will be used when -- executed - e.g. scripts and software installs have an execution_id, and @@ -42,7 +42,10 @@ CREATE TABLE upcoming_activities ( PRIMARY KEY (id), UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), - INDEX idx_upcoming_activities_host_id_activity_type (host_id, priority, created_at, activity_type), + -- index for the common access pattern to get the next activity to execute + INDEX idx_upcoming_activities_host_id_priority_created_at (host_id, priority, created_at), + -- index for the common access pattern to get by activity type (e.g. deleting pending scripts) + INDEX idx_upcoming_activities_host_id_activity_type (activity_type, host_id), CONSTRAINT fk_upcoming_activities_user_id FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci @@ -81,6 +84,38 @@ CREATE TABLE script_upcoming_activities ( CONSTRAINT fk_script_upcoming_activities_setup_experience_script_id FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return err + } + + _, err = tx.Exec(` +CREATE TABLE software_install_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, + + -- those are all columns and not JSON fields because we need FKs on them to + -- do processing ON DELETE, otherwise we'd have to check for existence of + -- each one when executing the activity (we need the enqueue next activity + -- action to be efficient). + software_installer_id INT UNSIGNED NULL, + policy_id INT UNSIGNED NULL, + software_title_id INT UNSIGNED NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_software_install_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_software_installer_id + FOREIGN KEY (software_installer_id) REFERENCES software_installers (id) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_policy_id + FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, + CONSTRAINT fk_software_install_upcoming_activities_software_title_id + FOREIGN KEY (software_title_id) REFERENCES software_titles (id) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci `, ) if err != nil { diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index e67983223ee3..d006306da2f3 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1819,6 +1819,25 @@ CREATE TABLE `software_host_counts` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `software_install_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `software_installer_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `software_title_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_software_install_upcoming_activities_software_installer_id` (`software_installer_id`), + KEY `fk_software_install_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_software_install_upcoming_activities_software_title_id` (`software_title_id`), + CONSTRAINT `fk_software_install_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_software_install_upcoming_activities_software_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_software_title_id` FOREIGN KEY (`software_title_id`) REFERENCES `software_titles` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `software_installer_labels` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `software_installer_id` int unsigned NOT NULL, @@ -1939,14 +1958,15 @@ CREATE TABLE `upcoming_activities` ( `priority` int NOT NULL DEFAULT '0', `user_id` int unsigned DEFAULT NULL, `fleet_initiated` tinyint(1) NOT NULL DEFAULT '0', - `activity_type` enum('script','software_install','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, + `activity_type` enum('script','software_install','software_uninstall','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `payload` json NOT NULL, `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`id`), UNIQUE KEY `idx_upcoming_activities_execution_id` (`execution_id`), - KEY `idx_upcoming_activities_host_id_activity_type` (`host_id`,`priority`,`created_at`,`activity_type`), + KEY `idx_upcoming_activities_host_id_priority_created_at` (`host_id`,`priority`,`created_at`), + KEY `idx_upcoming_activities_host_id_activity_type` (`activity_type`,`host_id`), KEY `fk_upcoming_activities_user_id` (`user_id`), CONSTRAINT `fk_upcoming_activities_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; From 742f29c5ede65d6299beb97d20468ae141f171f1 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 15:20:53 -0500 Subject: [PATCH 06/15] Fix list pending scripts for host ordering --- server/datastore/mysql/scripts.go | 31 ++++++++++++++++++-------- server/datastore/mysql/scripts_test.go | 2 ++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 3cb454f25891..e1f24bc7faf3 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -278,6 +278,14 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { // pending host script executions are those without results in // host_script_results UNION those in the upcoming activities queue + // + // Note that: + // > Use of ORDER BY for individual SELECT statements implies nothing about + // > the order in which the rows appear in the final result because UNION by + // > default produces an unordered set of rows. + // + // So we need to order the final result set. + internalWhere := "" if onlyShowInternal { internalWhere = " AND is_internal = TRUE" @@ -288,15 +296,16 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID id, host_id, execution_id, - script_id + script_id, + 1 as topmost, + 0 as priority, + created_at FROM host_script_results WHERE host_id = ? AND %s - %s - ORDER BY - created_at ASC`, whereFilterPendingScript, internalWhere) + %s`, whereFilterPendingScript, internalWhere) if onlyShowInternal { internalWhere = " AND JSON_EXTRACT(payload, '$.is_internal') = 1" @@ -306,7 +315,10 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID ua.id, ua.host_id, ua.execution_id, - sua.script_id + sua.script_id, + 0 as topmost, + ua.priority, + ua.created_at FROM upcoming_activities ua INNER JOIN script_upcoming_activities sua @@ -318,11 +330,12 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) ) - %s - ORDER BY - ua.priority DESC, ua.created_at ASC`, internalWhere) + %s`, internalWhere) - stmt := fmt.Sprintf(`(%s) UNION (%s)`, listHSRStmt, listUAStmt) + stmt := fmt.Sprintf(` + SELECT id, host_id, execution_id, script_id + FROM ( (%s) UNION (%s) ) as t + ORDER BY topmost DESC, priority DESC, created_at ASC`, listHSRStmt, listUAStmt) var results []*fleet.HostScriptResult seconds := int(constants.MaxServerWaitTime.Seconds()) diff --git a/server/datastore/mysql/scripts_test.go b/server/datastore/mysql/scripts_test.go index b3c83e08e8a4..c19ed4a35385 100644 --- a/server/datastore/mysql/scripts_test.go +++ b/server/datastore/mysql/scripts_test.go @@ -136,6 +136,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { */ // create another script execution request (null user id this time) + time.Sleep(time.Millisecond) // ensure a different timestamp createdScript2, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo2", @@ -199,6 +200,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { */ // create an async execution request + time.Sleep(time.Millisecond) // ensure a different timestamp createdScript3, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo 3", From 0fa9bb31a0cc64b3e75b57d7d8da944ebb87f8c8 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 Jan 2025 16:22:01 -0500 Subject: [PATCH 07/15] Implement inserting pending host software installs, list pending, test --- ee/server/service/setup_experience.go | 1 + server/datastore/mysql/software_installers.go | 125 +++++++++++++----- .../mysql/software_installers_test.go | 116 +++++++++------- 3 files changed, 160 insertions(+), 82 deletions(-) diff --git a/ee/server/service/setup_experience.go b/ee/server/service/setup_experience.go index 79698a67e28d..7a75bca9738d 100644 --- a/ee/server/service/setup_experience.go +++ b/ee/server/service/setup_experience.go @@ -178,6 +178,7 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string case len(installersPending) > 0: // enqueue installers for _, installer := range installersPending { + // TODO(mna): this should be top priority as this is setup exp. installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, false, nil) if err != nil { return false, ctxerr.Wrap(ctx, err, "queueing setup experience install request") diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 0403d88bdcff..d2bf8884476d 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -18,21 +18,37 @@ import ( ) func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { - // TODO(mna): must union with upcoming queue const stmt = ` - SELECT - execution_id - FROM - host_software_installs - WHERE - host_id = ? - AND - status = ? - ORDER BY - created_at ASC + SELECT + execution_id + FROM ( + SELECT + execution_id, + 1 as topmost, + 0 as priority, + created_at + FROM + host_software_installs + WHERE + host_id = ? AND + status = ? + UNION + SELECT + execution_id, + 0 as topmost, + priority, + created_at + FROM + upcoming_activities + WHERE + host_id = ? AND + activity_type = 'software_install' + ) as t + ORDER BY topmost DESC, priority ASC, created_at ASC ` var results []string - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID, fleet.SoftwareInstallPending); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, + stmt, hostID, fleet.SoftwareInstallPending, hostID); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending software installs") } return results, nil @@ -695,15 +711,37 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, } func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { - // TODO(mna): must insert in upcoming queue const ( - getInstallerStmt = `SELECT filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name - FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` - insertStmt = ` - INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` + getInstallerStmt = ` +SELECT + filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name +FROM + software_installers si + LEFT JOIN software_titles st + ON si.title_id = st.id +WHERE si.id = ?` + + //(execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'software_install', ?, + JSON_OBJECT( + 'self_service', ?, + 'installer_filename', ?, + 'version', ?, + 'software_title_name', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + ) + )` + + insertSIUAStmt = ` +INSERT INTO software_install_upcoming_activities + (upcoming_activity_id, software_installer_id, policy_id, software_title_id) +VALUES + (?, ?, ?, ?)` + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -733,24 +771,43 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui } var userID *uint + fleetInitiated := true if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { userID = &ctxUser.ID + fleetInitiated = false } - installID := uuid.NewString() - _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - installID, - hostID, - softwareInstallerID, - userID, - selfService, - policyID, - installerDetails.Filename, - installerDetails.Version, - installerDetails.TitleID, - installerDetails.TitleName, - ) + execID := uuid.NewString() - return installID, ctxerr.Wrap(ctx, err, "inserting new install software request") + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := ds.writer(ctx).ExecContext(ctx, insertUAStmt, + hostID, + 0, // TODO(mna): detect if this is software install for setup exp, to boost priority + userID, + fleetInitiated, + execID, + selfService, + installerDetails.Filename, + installerDetails.Version, + installerDetails.TitleName, + userID, + ) + if err != nil { + return err + } + + activityID, _ := res.LastInsertId() + _, err = ds.writer(ctx).ExecContext(ctx, insertSIUAStmt, + activityID, + softwareInstallerID, + policyID, + installerDetails.TitleID, + ) + if err != nil { + return err + } + return nil + }) + return execID, ctxerr.Wrap(ctx, err, "inserting new install software request") } func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 56d996ebe59e..d1fcc679d57b 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -129,77 +129,97 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall3, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, false, nil) require.NoError(t, err) + time.Sleep(time.Millisecond) hostInstall4, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host2.ID, - InstallUUID: hostInstall4, - InstallScriptExitCode: ptr.Int(0), - }) + pendingHost1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) require.NoError(t, err) + require.Equal(t, 2, len(pendingHost1)) + require.Equal(t, hostInstall1, pendingHost1[0]) + require.Equal(t, hostInstall2, pendingHost1[1]) - hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + pendingHost2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) require.NoError(t, err) + require.Equal(t, 2, len(pendingHost2)) + require.Equal(t, hostInstall3, pendingHost2[0]) + require.Equal(t, hostInstall4, pendingHost2[1]) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host2.ID, - InstallUUID: hostInstall5, - PreInstallConditionOutput: ptr.String(""), // pre-install query did not return results, so install failed - }) - require.NoError(t, err) + _ = installerID3 - installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) - require.NoError(t, err) - require.Equal(t, 2, len(installDetailsList1)) + // TODO(mna): uncomment the rest of this test once execution of upcoming activities is implemented + /* + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall4, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) - installDetailsList2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) - require.NoError(t, err) - require.Equal(t, 1, len(installDetailsList2)) + hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall5, + PreInstallConditionOutput: ptr.String(""), // pre-install query did not return results, so install failed + }) + require.NoError(t, err) - require.Contains(t, installDetailsList1, hostInstall1) - require.Contains(t, installDetailsList1, hostInstall2) + installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Equal(t, 2, len(installDetailsList1)) - require.Contains(t, installDetailsList2, hostInstall3) + installDetailsList2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) + require.NoError(t, err) + require.Equal(t, 1, len(installDetailsList2)) - exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) - require.NoError(t, err) + require.Contains(t, installDetailsList1, hostInstall1) + require.Contains(t, installDetailsList1, hostInstall2) - require.Equal(t, host1.ID, exec1.HostID) - require.Equal(t, hostInstall1, exec1.ExecutionID) - require.Equal(t, "hello DUCKY", exec1.InstallScript) - require.Equal(t, "world BIRD", exec1.PostInstallScript) - require.Equal(t, installerID1, exec1.InstallerID) - require.Equal(t, "SELECT 1", exec1.PreInstallCondition) - require.False(t, exec1.SelfService) - assert.Equal(t, "goodbye MONSTER", exec1.UninstallScript) + require.Contains(t, installDetailsList2, hostInstall3) - hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, true, nil) - require.NoError(t, err) + exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) + require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: host1.ID, - InstallUUID: hostInstall6, - PreInstallConditionOutput: ptr.String("output"), - }) - require.NoError(t, err) + require.Equal(t, host1.ID, exec1.HostID) + require.Equal(t, hostInstall1, exec1.ExecutionID) + require.Equal(t, "hello DUCKY", exec1.InstallScript) + require.Equal(t, "world BIRD", exec1.PostInstallScript) + require.Equal(t, installerID1, exec1.InstallerID) + require.Equal(t, "SELECT 1", exec1.PreInstallCondition) + require.False(t, exec1.SelfService) + assert.Equal(t, "goodbye MONSTER", exec1.UninstallScript) - exec2, err := ds.GetSoftwareInstallDetails(ctx, hostInstall6) - require.NoError(t, err) + hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, true, nil) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host1.ID, + InstallUUID: hostInstall6, + PreInstallConditionOutput: ptr.String("output"), + }) + require.NoError(t, err) + + exec2, err := ds.GetSoftwareInstallDetails(ctx, hostInstall6) + require.NoError(t, err) - require.Equal(t, host1.ID, exec2.HostID) - require.Equal(t, hostInstall6, exec2.ExecutionID) - require.Equal(t, "banana", exec2.InstallScript) - require.Equal(t, "apple", exec2.PostInstallScript) - require.Equal(t, installerID3, exec2.InstallerID) - require.Equal(t, "SELECT 3", exec2.PreInstallCondition) - require.True(t, exec2.SelfService) + require.Equal(t, host1.ID, exec2.HostID) + require.Equal(t, hostInstall6, exec2.ExecutionID) + require.Equal(t, "banana", exec2.InstallScript) + require.Equal(t, "apple", exec2.PostInstallScript) + require.Equal(t, installerID3, exec2.InstallerID) + require.Equal(t, "SELECT 3", exec2.PreInstallCondition) + require.True(t, exec2.SelfService) + */ } func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { From 854ecd07edc0f19aecec611111f950aef23681a1 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 09:15:06 -0500 Subject: [PATCH 08/15] Fix db migrations order, rebase on feature branch --- ...QueueTable.go => 20250113162751_AddUnifiedQueueTable.go} | 6 +++--- server/datastore/mysql/schema.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename server/datastore/mysql/migrations/tables/{20250106162751_AddUnifiedQueueTable.go => 20250113162751_AddUnifiedQueueTable.go} (97%) diff --git a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go similarity index 97% rename from server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go rename to server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go index 1100b8196500..749e8e66536d 100644 --- a/server/datastore/mysql/migrations/tables/20250106162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go @@ -6,10 +6,10 @@ import ( ) func init() { - MigrationClient.AddMigration(Up_20250106162751, Down_20250106162751) + MigrationClient.AddMigration(Up_20250113162751, Down_20250113162751) } -func Up_20250106162751(tx *sql.Tx) error { +func Up_20250113162751(tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE upcoming_activities ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, @@ -124,6 +124,6 @@ CREATE TABLE software_install_upcoming_activities ( return nil } -func Down_20250106162751(tx *sql.Tx) error { +func Down_20250113162751(tx *sql.Tx) error { return nil } diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index d006306da2f3..e1def8d809e0 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1113,7 +1113,7 @@ CREATE TABLE `migration_status_tables` ( PRIMARY KEY (`id`) ) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=349 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250106162751,1,'2020-01-01 01:01:01'),(347,20250107165731,1,'2020-01-01 01:01:01'),(348,20250109150150,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250107165731,1,'2020-01-01 01:01:01'),(347,20250109150150,1,'2020-01-01 01:01:01'),(348,20250113162751,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( From 5c24775a7004c3b060c7cbe61892d4ef4f29b49d Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 12:29:07 -0500 Subject: [PATCH 09/15] Update some todos --- server/datastore/mysql/scripts.go | 18 +- server/datastore/mysql/software_installers.go | 202 +++++++++++++++--- 2 files changed, 177 insertions(+), 43 deletions(-) diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index e1f24bc7faf3..16a58785c886 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -66,7 +66,7 @@ func newHostScriptExecutionRequest(ctx context.Context, tx sqlx.ExtContext, requ INSERT INTO upcoming_activities (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) VALUES - (?, ?, ?, ?, 'script', ?, + (?, ?, ?, ?, 'script', ?, JSON_OBJECT( 'sync_request', ?, 'is_internal', ?, @@ -75,7 +75,7 @@ VALUES )` insSUAStmt = ` -INSERT INTO script_upcoming_activities +INSERT INTO script_upcoming_activities (upcoming_activity_id, script_id, script_content_id, policy_id, setup_experience_script_id) VALUES (?, ?, ?, ?, ?) @@ -109,7 +109,7 @@ WHERE request.HostID, priority, request.UserID, - false, // TODO(mna): do we have fleet-initiated scripts? + isInternal || request.UserID == nil, // TODO(mna): confirm if this makes sense execID, request.SyncRequest, isInternal, @@ -320,7 +320,7 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID ua.priority, ua.created_at FROM - upcoming_activities ua + upcoming_activities ua INNER JOIN script_upcoming_activities sua ON ua.id = sua.upcoming_activity_id WHERE @@ -333,8 +333,8 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID %s`, internalWhere) stmt := fmt.Sprintf(` - SELECT id, host_id, execution_id, script_id - FROM ( (%s) UNION (%s) ) as t + SELECT id, host_id, execution_id, script_id + FROM ( (%s) UNION (%s) ) as t ORDER BY topmost DESC, priority DESC, created_at ASC`, listHSRStmt, listUAStmt) var results []*fleet.HostScriptResult @@ -355,7 +355,9 @@ func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, host_id = ? AND script_id = ? AND exit_code IS NULL + UNION + SELECT 1 FROM @@ -628,11 +630,11 @@ func (ds *Datastore) deletePendingHostScriptExecutionsForPolicy(ctx context.Cont deleteUAStmt := ` DELETE FROM - upcoming_activities + upcoming_activities USING upcoming_activities INNER JOIN script_upcoming_activities sua - ON upcoming_activities.id = sua.upcoming_activity_id + ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' AND sua.policy_id = ? AND diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index d2bf8884476d..a03b12d5a1d5 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -19,7 +19,7 @@ import ( func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { const stmt = ` - SELECT + SELECT execution_id FROM ( SELECT @@ -32,7 +32,9 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin WHERE host_id = ? AND status = ? - UNION + + UNION + SELECT execution_id, 0 as topmost, @@ -43,7 +45,7 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin WHERE host_id = ? AND activity_type = 'software_install' - ) as t + ) as t ORDER BY topmost DESC, priority ASC, created_at ASC ` var results []string @@ -55,7 +57,6 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin } func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { - // TODO(mna): must also look in upcoming queue const stmt = ` SELECT hsi.host_id AS host_id, @@ -81,7 +82,39 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId script_contents pisnt ON pisnt.id = si.post_install_script_content_id WHERE - hsi.execution_id = ?` + hsi.execution_id = ? + + UNION + + SELECT + ua.host_id AS host_id, + ua.execution_id AS execution_id, + siua.software_installer_id AS installer_id, + JSON_EXTRACT(ua.payload, '$.self_service') AS self_service, + COALESCE(si.pre_install_query, '') AS pre_install_condition, + inst.cot could be nulntents AS install_script, + uninst.contents AS uninstall_script, + COALESCE(pisnt.contents, '') AS post_install_script + FROM + upcoming_activities ua + INNER JOIN + software_install_upcoming_activities siua + ON ua.id = siua.upcoming_activity_id + INNER JOIN + software_installers si + ON siua.software_installer_id = si.id + LEFT OUTER JOIN + script_contents inst + ON inst.id = si.install_script_content_id + LEFT OUTER JOIN + script_contents uninst + ON uninst.id = si.uninstall_script_content_id + LEFT OUTER JOIN + script_contents pisnt + ON pisnt.id = si.post_install_script_content_id + WHERE + ua.execution_id = ? +` result := &fleet.SoftwareInstallDetails{} if err := sqlx.GetContext(ctx, ds.reader(ctx), result, stmt, executionId); err != nil { @@ -474,8 +507,9 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } func (ds *Datastore) ValidateOrbitSoftwareInstallerAccess(ctx context.Context, hostID uint, installerID uint) (bool, error) { - // TODO(mna): this is ok to only look in host_software_installs, because orbit should not - // be able to get the installer until it is ready to install. + // NOTE: this is ok to only look in host_software_installs (and ignore + // upcoming_activities), because orbit should not be able to get the + // installer until it is ready to install. query := ` SELECT 1 FROM @@ -684,14 +718,18 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error }) } -// deletePendingSoftwareInstallsForPolicy should be called after a policy is deleted to remove any pending software installs +// deletePendingSoftwareInstallsForPolicy should be called after a policy is +// deleted to remove any pending software installs func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, teamID *uint, policyID uint) error { var globalOrTeamID uint if teamID != nil { globalOrTeamID = *teamID } - // TODO(mna): must also delete from upcoming queue + // NOTE(mna): I'm adding the deletion for the upcoming_activities too, but I + // don't think the existing code works as intended anyway as the + // host_software_installs.policy_id column has a ON DELETE SET NULL foreign + // key, so the deletion statement will not find any row. const deleteStmt = ` DELETE FROM host_software_installs @@ -707,6 +745,25 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, return ctxerr.Wrap(ctx, err, "delete pending software installs for policy") } + const deleteUAStmt = ` + DELETE FROM + upcoming_activities + USING + upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE + ua.activity_type = 'software_install' AND + siua.policy_id = ? AND + siua.software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) + ` + _, err = ds.writer(ctx).ExecContext(ctx, deleteUAStmt, policyID, globalOrTeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete upcoming software installs for policy") + } + return nil } @@ -721,7 +778,6 @@ FROM ON si.title_id = st.id WHERE si.id = ?` - //(execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) insertUAStmt = ` INSERT INTO upcoming_activities (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) @@ -779,7 +835,7 @@ VALUES execID := uuid.NewString() err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := ds.writer(ctx).ExecContext(ctx, insertUAStmt, + res, err := tx.ExecContext(ctx, insertUAStmt, hostID, 0, // TODO(mna): detect if this is software install for setup exp, to boost priority userID, @@ -796,7 +852,7 @@ VALUES } activityID, _ := res.LastInsertId() - _, err = ds.writer(ctx).ExecContext(ctx, insertSIUAStmt, + _, err = tx.ExecContext(ctx, insertSIUAStmt, activityID, softwareInstallerID, policyID, @@ -818,9 +874,6 @@ func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, inst func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Context, tx sqlx.ExtContext, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { if wasMetadataUpdated || wasPackageUpdated { // cancel pending installs/uninstalls - // TODO(mna): this deletes from host_script_results, but is actually related to software installs. - // Must add deletion from upcoming queue. - // TODO make this less naive; this assumes that installs/uninstalls execute and report back immediately _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE software_installer_id = ? AND status = 'pending_uninstall' @@ -834,6 +887,16 @@ func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Cont if err != nil { return ctxerr.Wrap(ctx, err, "delete pending host software installs/uninstalls") } + + _, err = tx.ExecContext(ctx, `DELETE FROM upcoming_activities + USING + upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE siua.software_installer_id = ? AND activity_type IN ('software_install', 'software_uninstall')`, installerID) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete upcoming host software installs/uninstalls") + } } if wasPackageUpdated { // hide existing install counts @@ -848,15 +911,32 @@ func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Cont } func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { - // TODO(mna): must insert in upcoming queue const ( getInstallerStmt = `SELECT title_id, COALESCE(st.name, '[deleted title]') title_name FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` - insertStmt = ` - INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id, user_id, uninstall, installer_filename, software_title_id, software_title_name, version) - VALUES (?, ?, ?, ?, 1, '', ?, ?, 'unknown') - ` + + // TODO(mna): To be reviewed once software uninstsall is better understood, + // as it is it wouldn't work because the same execution_id is used to + // insert a script execution request and the software uninstall request. + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'software_uninstall', ?, + JSON_OBJECT( + 'installer_filename', '', + 'version', 'unknown', + 'software_title_name', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + ) + )` + + insertSIUAStmt = ` +INSERT INTO software_install_upcoming_activities + (upcoming_activity_id, software_installer_id, software_title_id) +VALUES + (?, ?, ?)` + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -883,23 +963,42 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi } var userID *uint + fleetInitiated := true if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { userID = &ctxUser.ID + fleetInitiated = false } - _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - executionID, - hostID, - softwareInstallerID, - userID, - installerDetails.TitleID, - installerDetails.TitleName, - ) + + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, insertUAStmt, + hostID, + 0, // Uninstalls are never used in setup experience, so always default priority + userID, + fleetInitiated, + executionID, + installerDetails.TitleName, + userID, + ) + if err != nil { + return err + } + + activityID, _ := res.LastInsertId() + _, err = tx.ExecContext(ctx, insertSIUAStmt, + activityID, + softwareInstallerID, + installerDetails.TitleID, + ) + if err != nil { + return err + } + return nil + }) return ctxerr.Wrap(ctx, err, "inserting new uninstall software request") } func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { - // TODO(mna): must also look in upcoming queue query := ` SELECT hsi.execution_id AS execution_id, @@ -924,7 +1023,37 @@ FROM LEFT JOIN software_titles st ON hsi.software_title_id = st.id WHERE hsi.execution_id = :execution_id - ` + +UNION + +SELECT + ua.execution_id AS execution_id, + NULL AS pre_install_query_output, + NULL AS post_install_script_output, + NULL AS install_script_output, + ua.host_id AS host_id, + COALESCE(st.name, JSON_EXTRACT(ua.payload, '$.software_title_name')) AS software_title, + siua.software_title_id, + 'pending_install' AS status, + JSON_EXTRACT(ua.payload, '$.installer_filename') AS software_package, + ua.user_id AS user_id, + NULL AS post_install_script_exit_code, + NULL AS install_script_exit_code, + JSON_EXTRACT(ua.payload, '$.self_service') AS self_service, + NULL AS host_deleted_at, + siua.policy_id AS policy_id, + ua.created_at as created_at, + ua.updated_at as updated_at +FROM + upcoming_activities ua + INNER JOIN software_install_upcoming_activities siua + ON ua.id = siua.upcoming_activity_id + LEFT JOIN software_titles st + ON siua.software_title_id = st.id +WHERE + ua.execution_id = :execution_id AND + ua.activity_type = 'software_install' +` stmt, args, err := sqlx.Named(query, map[string]any{ "execution_id": resultsUUID, @@ -948,7 +1077,8 @@ WHERE func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { var dest fleet.SoftwareInstallerStatusSummary - // TODO(mna): must also look in upcoming queue for pending + // TODO(mna): must also look in upcoming queue for pending, and most recent + // attempt for an installer might be in upcoming queue... stmt := ` SELECT COALESCE(SUM( IF(status = :software_status_pending_install, 1, 0)), 0) AS pending_install, @@ -973,7 +1103,8 @@ WHERE AND host_deleted_at IS NULL AND removed = 0 GROUP BY - host_id)) s` + host_id) +) s` query, args, err := sqlx.Named(stmt, map[string]interface{}{ "installer_id": installerID, @@ -1055,7 +1186,8 @@ func (ds *Datastore) softwareInstallerJoin(installerID uint, status fleet.Softwa if status2 != "" { statusFilter = "hsi.status IN (:status, :status2)" } - // TODO(mna): must join with upcoming queue for pending + // TODO(mna): must join with upcoming queue for pending, the "most recent install attempt" + // could be in upcoming queue (in which case this impacts also the non-pending status) stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -1084,7 +1216,7 @@ WHERE func (ds *Datastore) GetHostLastInstallData(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { // TODO(mna): I think that if there's none in host_software_installs, must take - // latest in upcoming queue. + // latest in upcoming queue (latest attempt might actually be in upcoming). stmt := ` SELECT execution_id, hsi.status FROM host_software_installs hsi From e3d5636aa2c49fb506f7b7a1523269741eb416e1 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 15:28:39 -0500 Subject: [PATCH 10/15] Add activated_at, all pending states are now considered always in upcoming activities --- server/datastore/mysql/activities.go | 19 +++++-- .../20250113162751_AddUnifiedQueueTable.go | 1 + server/datastore/mysql/schema.sql | 1 + server/datastore/mysql/scripts.go | 57 ++----------------- server/datastore/mysql/software_installers.go | 9 ++- 5 files changed, 30 insertions(+), 57 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 99b342d283c8..1e49741e0f55 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -252,6 +252,13 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint // TODO(mna): use the upcoming queue instead (but also include in-execution stuff // that have no results yet)? countStmts := []string{ + // everything in upcoming_activities is pending for that host + `SELECT + COUNT(*) c + FROM upcoming_activities + WHERE host_id = :host_id`, + + // scripts without result that are not "uninstall scripts" for a software install are pending `SELECT COUNT(*) c FROM host_script_results hsr @@ -261,23 +268,27 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint exit_code IS NULL AND hsi.execution_id IS NULL AND (sync_request = 0 OR hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND))`, + + // software installs without result are pending `SELECT COUNT(*) c FROM host_software_installs hsi WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND hsi.status = :software_status_install_pending`, + + // software uninstalls without result are pending `SELECT COUNT(*) c FROM host_software_installs hsi WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND hsi.status = :software_status_uninstall_pending`, - ` - SELECT + + // VPP installs without command result are pending + `SELECT COUNT(*) c FROM nano_view_queue nvq JOIN host_vpp_software_installs hvsi ON nvq.command_uuid = hvsi.command_uuid - WHERE hvsi.host_id = :host_id AND nvq.status IS NULL - `, + WHERE hvsi.host_id = :host_id AND nvq.status IS NULL`, } var count uint diff --git a/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go index 749e8e66536d..ce32c40dfff7 100644 --- a/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go +++ b/server/datastore/mysql/migrations/tables/20250113162751_AddUnifiedQueueTable.go @@ -37,6 +37,7 @@ CREATE TABLE upcoming_activities ( payload JSON NOT NULL, -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + activated_at DATETIME(6) NULL, created_at DATETIME(6) NOT NULL DEFAULT NOW(6), updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index e1def8d809e0..141590f1cd45 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1961,6 +1961,7 @@ CREATE TABLE `upcoming_activities` ( `activity_type` enum('script','software_install','software_uninstall','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `payload` json NOT NULL, + `activated_at` datetime(6) DEFAULT NULL, `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`id`), diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 16a58785c886..460b60677fdf 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -276,49 +276,19 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f } func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { - // pending host script executions are those without results in - // host_script_results UNION those in the upcoming activities queue - // - // Note that: - // > Use of ORDER BY for individual SELECT statements implies nothing about - // > the order in which the rows appear in the final result because UNION by - // > default produces an unordered set of rows. - // - // So we need to order the final result set. - internalWhere := "" - if onlyShowInternal { - internalWhere = " AND is_internal = TRUE" - } - - listHSRStmt := fmt.Sprintf(` - SELECT - id, - host_id, - execution_id, - script_id, - 1 as topmost, - 0 as priority, - created_at - FROM - host_script_results - WHERE - host_id = ? AND - %s - %s`, whereFilterPendingScript, internalWhere) - if onlyShowInternal { internalWhere = " AND JSON_EXTRACT(payload, '$.is_internal') = 1" } - listUAStmt := fmt.Sprintf(` + listStmt := fmt.Sprintf(` SELECT ua.id, ua.host_id, ua.execution_id, sua.script_id, - 0 as topmost, ua.priority, - ua.created_at + ua.created_at, + IF(ua.activated_at IS NULL, 0, 1) AS topmost FROM upcoming_activities ua INNER JOIN script_upcoming_activities sua @@ -330,16 +300,12 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) ) - %s`, internalWhere) - - stmt := fmt.Sprintf(` - SELECT id, host_id, execution_id, script_id - FROM ( (%s) UNION (%s) ) as t - ORDER BY topmost DESC, priority DESC, created_at ASC`, listHSRStmt, listUAStmt) + %s + ODER BY topmost DESC, priority DESC, created_at ASC`, internalWhere) var results []*fleet.HostScriptResult seconds := int(constants.MaxServerWaitTime.Seconds()) - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID, seconds, hostID, seconds); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, listStmt, hostID, seconds, hostID, seconds); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending host script executions") } return results, nil @@ -347,17 +313,6 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, scriptID uint) (bool, error) { const getStmt = ` - SELECT - 1 - FROM - host_script_results - WHERE - host_id = ? AND - script_id = ? AND - exit_code IS NULL - - UNION - SELECT 1 FROM diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index a03b12d5a1d5..5c626ef37074 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -918,6 +918,11 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi // TODO(mna): To be reviewed once software uninstsall is better understood, // as it is it wouldn't work because the same execution_id is used to // insert a script execution request and the software uninstall request. + // Although I think one way to solve this would be to enqueue an uninstall + // activity in upcoming_activities, and when it's ready to run, insert in + // both host_script_results and host_software_installs, as it does today. + // So while it's pending, it's a single row in upcoming_activities, and when + // it's about to run, it's exactly the same as today. insertUAStmt = ` INSERT INTO upcoming_activities (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) @@ -1046,9 +1051,9 @@ SELECT ua.updated_at as updated_at FROM upcoming_activities ua - INNER JOIN software_install_upcoming_activities siua + INNER JOIN software_install_upcoming_activities siua ON ua.id = siua.upcoming_activity_id - LEFT JOIN software_titles st + LEFT JOIN software_titles st ON siua.software_title_id = st.id WHERE ua.execution_id = :execution_id AND From 61586648d9c9b0bd93b2f299ae0a238d6ed5b608 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 15:32:27 -0500 Subject: [PATCH 11/15] Fix query --- server/datastore/mysql/scripts.go | 51 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 460b60677fdf..8aded2f3b9f7 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -282,30 +282,37 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID } listStmt := fmt.Sprintf(` SELECT - ua.id, - ua.host_id, - ua.execution_id, - sua.script_id, - ua.priority, - ua.created_at, - IF(ua.activated_at IS NULL, 0, 1) AS topmost - FROM - upcoming_activities ua - INNER JOIN script_upcoming_activities sua - ON ua.id = sua.upcoming_activity_id - WHERE - ua.host_id = ? AND - ua.activity_type = 'script' AND - ( - JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR - ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) - ) - %s - ODER BY topmost DESC, priority DESC, created_at ASC`, internalWhere) + id, + host_id, + execution_id, + script_id, + created_at + FROM ( + SELECT + ua.id, + ua.host_id, + ua.execution_id, + sua.script_id, + ua.priority, + ua.created_at, + IF(ua.activated_at IS NULL, 0, 1) AS topmost + FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id + WHERE + ua.host_id = ? AND + ua.activity_type = 'script' AND + ( + JSON_EXTRACT(ua.payload, '$.sync_request') = 0 OR + ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) + ) + %s + ORDER BY topmost DESC, priority DESC, created_at ASC) t`, internalWhere) var results []*fleet.HostScriptResult seconds := int(constants.MaxServerWaitTime.Seconds()) - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, listStmt, hostID, seconds, hostID, seconds); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, listStmt, hostID, seconds); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending host script executions") } return results, nil @@ -326,7 +333,7 @@ func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, ` var results []*uint - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, getStmt, hostID, scriptID, hostID, scriptID); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, getStmt, hostID, scriptID); err != nil { return false, ctxerr.Wrap(ctx, err, "is execution pending for host") } return len(results) > 0, nil From 2c3d5935699af303381220767873c75c438b9a53 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 15:46:46 -0500 Subject: [PATCH 12/15] Adjust software installers queries for pending/activated activities --- server/datastore/mysql/software_installers.go | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 5c626ef37074..74f37454825c 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -24,20 +24,7 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin FROM ( SELECT execution_id, - 1 as topmost, - 0 as priority, - created_at - FROM - host_software_installs - WHERE - host_id = ? AND - status = ? - - UNION - - SELECT - execution_id, - 0 as topmost, + IF(activated_at IS NULL, 0, 1) as topmost, priority, created_at FROM @@ -45,12 +32,10 @@ func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uin WHERE host_id = ? AND activity_type = 'software_install' - ) as t - ORDER BY topmost DESC, priority ASC, created_at ASC + ORDER BY topmost DESC, priority ASC, created_at ASC) as t ` var results []string - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, - stmt, hostID, fleet.SoftwareInstallPending, hostID); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending software installs") } return results, nil @@ -113,7 +98,8 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId script_contents pisnt ON pisnt.id = si.post_install_script_content_id WHERE - ua.execution_id = ? + ua.execution_id = ? AND + ua.activated_at IS NULL -- if already activated, then it is covered by the other SELECT ` result := &fleet.SoftwareInstallDetails{} @@ -1057,7 +1043,8 @@ FROM ON siua.software_title_id = st.id WHERE ua.execution_id = :execution_id AND - ua.activity_type = 'software_install' + ua.activity_type = 'software_install' AND + ua.activated_at IS NULL -- if already activated, covered by the other SELECT ` stmt, args, err := sqlx.Named(query, map[string]any{ From a5e0b8f6c50cae9110862849113557cadf345c76 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 13 Jan 2025 16:46:48 -0500 Subject: [PATCH 13/15] Adjust the queries for the list host upcoming activities, incomplete --- server/datastore/mysql/activities.go | 273 ++++++++---------- server/datastore/mysql/scripts.go | 2 +- server/datastore/mysql/software_installers.go | 4 +- 3 files changed, 118 insertions(+), 161 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 1e49741e0f55..22c64f17864e 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/fleetdm/fleet/v4/pkg/scripts" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/go-kit/log/level" @@ -249,62 +248,13 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint // NOTE: Be sure to update both the count (here) and list statements (below) // if the query condition is modified. - // TODO(mna): use the upcoming queue instead (but also include in-execution stuff - // that have no results yet)? - countStmts := []string{ - // everything in upcoming_activities is pending for that host - `SELECT - COUNT(*) c - FROM upcoming_activities - WHERE host_id = :host_id`, - - // scripts without result that are not "uninstall scripts" for a software install are pending - `SELECT - COUNT(*) c - FROM host_script_results hsr - LEFT OUTER JOIN - host_software_installs hsi ON hsi.execution_id = hsr.execution_id - WHERE hsr.host_id = :host_id AND - exit_code IS NULL AND - hsi.execution_id IS NULL AND - (sync_request = 0 OR hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND))`, - - // software installs without result are pending - `SELECT - COUNT(*) c - FROM host_software_installs hsi - WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND - hsi.status = :software_status_install_pending`, - - // software uninstalls without result are pending - `SELECT - COUNT(*) c - FROM host_software_installs hsi - WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND - hsi.status = :software_status_uninstall_pending`, - - // VPP installs without command result are pending - `SELECT - COUNT(*) c - FROM nano_view_queue nvq - JOIN host_vpp_software_installs hvsi ON nvq.command_uuid = hvsi.command_uuid - WHERE hvsi.host_id = :host_id AND nvq.status IS NULL`, - } + const countStmt = `SELECT + COUNT(*) c + FROM upcoming_activities + WHERE host_id = ?` var count uint - countStmt := `SELECT SUM(c) FROM ( ` + strings.Join(countStmts, " UNION ALL ") + ` ) AS counts` - - seconds := int(scripts.MaxServerWaitTime.Seconds()) - countStmt, args, err := sqlx.Named(countStmt, map[string]any{ - "host_id": hostID, - "max_wait_time": seconds, - "software_status_install_pending": fleet.SoftwareInstallPending, - "software_status_uninstall_pending": fleet.SoftwareUninstallPending, - }) - if err != nil { - return nil, nil, ctxerr.Wrap(ctx, err, "build count query from named args") - } - if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, args...); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, hostID); err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "count upcoming activities") } if count == 0 { @@ -319,154 +269,161 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint listStmts := []string{ // list pending scripts `SELECT - hsr.execution_id as uuid, - IF(hsr.policy_id IS NOT NULL, 'Fleet', u.name) as name, + ua.execution_id as uuid, + IF(sua.policy_id IS NOT NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) as name, -- TODO: should that use fleet_initiated u.id as user_id, - u.gravatar_url as gravatar_url, - u.email as user_email, + COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, + COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, :ran_script_type as activity_type, - hsr.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsr.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), 'script_name', COALESCE(ses.name, COALESCE(scr.name, '')), - 'script_execution_id', hsr.execution_id, - 'async', NOT hsr.sync_request, - 'policy_id', hsr.policy_id, - 'policy_name', p.name - ) as details + 'script_execution_id', ua.execution_id, + 'async', NOT JSON_EXTRACT(ua.payload, '$.sync_request), + 'policy_id', sua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, -- also, cancellable if topmost = 1 + ua.priority as priority FROM - host_script_results hsr - LEFT OUTER JOIN - users u ON u.id = hsr.user_id + upcoming_activities ua + INNER JOIN + script_upcoming_activities sua ON sua.upcoming_activity_id = ua.id LEFT OUTER JOIN - policies p ON p.id = hsr.policy_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsr.host_id + policies p ON p.id = sua.policy_id LEFT OUTER JOIN - scripts scr ON scr.id = hsr.script_id + host_display_names hdn ON hdn.host_id = ua.host_id LEFT OUTER JOIN - host_software_installs hsi ON hsi.execution_id = hsr.execution_id + scripts scr ON scr.id = sua.script_id LEFT OUTER JOIN - setup_experience_scripts ses ON ses.id = hsr.setup_experience_script_id + setup_experience_scripts ses ON ses.id = sua.setup_experience_script_id WHERE - hsr.host_id = :host_id AND - hsr.exit_code IS NULL AND - ( - hsr.sync_request = 0 OR - hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND) - ) AND - hsi.execution_id IS NULL + ua.host_id = :host_id AND + ua.activity_type = 'script' `, // list pending software installs `SELECT - hsi.execution_id as uuid, + ua.execution_id as uuid, -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), -- so we mark those as "Fleet" - IF(hsi.user_id IS NULL AND NOT hsi.self_service, 'Fleet', u.name) AS name, - hsi.user_id as user_id, - u.gravatar_url as gravatar_url, - u.email AS user_email, + -- TODO: should we use fleet_initiated instead + IF(ua.user_id IS NULL AND NOT JSON_EXTRACT(ua.payload, '$.self_service), 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, + ua.user_id as user_id, + COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, + COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, :installed_software_type as activity_type, - hsi.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsi.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), - 'software_title', COALESCE(st.name, ''), - 'software_package', si.filename, - 'install_uuid', hsi.execution_id, - 'status', CAST(hsi.status AS CHAR), - 'self_service', hsi.self_service IS TRUE, - 'policy_id', hsi.policy_id, - 'policy_name', p.name - ) as details + 'software_title', COALESCE(st.name, JSON_EXTRACT(ua.payload, '$.software_title_name')), + 'software_package', COALESCE(si.filename, JSON_EXTRACT(ua.payload, '$.installer_filename')), + 'install_uuid', ua.execution_id, + 'status', 'pending_install', + 'self_service', JSON_EXTRACT(ua.payload, '$.self_service) IS TRUE, + 'policy_id', siua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, -- also, cancellable if topmost = 1 + ua.priority as priority FROM - host_software_installs hsi + upcoming_activities ua INNER JOIN - software_installers si ON si.id = hsi.software_installer_id + software_install_upcoming_activities siua ON siua.upcoming_activity_id = ua.id + LEFT OUTER JOIN + software_installers si ON si.id = siua.software_installer_id LEFT OUTER JOIN software_titles st ON st.id = si.title_id LEFT OUTER JOIN - users u ON u.id = hsi.user_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - policies p ON p.id = hsi.policy_id + policies p ON p.id = siua.policy_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsi.host_id + host_display_names hdn ON hdn.host_id = ua.host_id WHERE - hsi.host_id = :host_id AND - hsi.status = :software_status_install_pending + ua.host_id = :host_id AND + ua.activity_type = 'software_install' `, // list pending software uninstalls `SELECT - hsi.execution_id as uuid, + ua.execution_id as uuid, -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), -- so we mark those as "Fleet" - IF(hsi.user_id IS NULL AND NOT hsi.self_service, 'Fleet', u.name) AS name, - hsi.user_id as user_id, - u.gravatar_url as gravatar_url, - u.email AS user_email, + -- TODO: should we use fleet_initiated instead + IF(ua.user_id IS NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, + ua.user_id as user_id, + COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, + COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, :uninstalled_software_type as activity_type, - hsi.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsi.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), - 'software_title', COALESCE(st.name, ''), - 'script_execution_id', hsi.execution_id, - 'status', CAST(hsi.status AS CHAR), - 'policy_id', hsi.policy_id, - 'policy_name', p.name - ) as details + 'software_title', COALESCE(st.name, JSON_EXTRACT(ua.payload, '$.software_title_name')), + 'script_execution_id', ua.execution_id, + 'status', 'pending_uninstall', + 'policy_id', siua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, -- also, cancellable if topmost = 1 + ua.priority as priority FROM - host_software_installs hsi + upcoming_activities ua INNER JOIN - software_installers si ON si.id = hsi.software_installer_id + software_install_upcoming_activities siua ON siua.upcoming_activity_id = ua.id + LEFT OUTER JOIN + software_installers si ON si.id = siua.software_installer_id LEFT OUTER JOIN software_titles st ON st.id = si.title_id LEFT OUTER JOIN - users u ON u.id = hsi.user_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - policies p ON p.id = hsi.policy_id + policies p ON p.id = siua.policy_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsi.host_id + host_display_names hdn ON hdn.host_id = ua.host_id WHERE - hsi.host_id = :host_id AND - hsi.status = :software_status_uninstall_pending + ua.host_id = :host_id AND + activity_type = 'software_uninstall' + `, + // TODO(mna): complete the VPP apps UNION when VPP apps are ready + `SELECT + ua.execution_id AS uuid, + IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, + u.id AS user_id, + COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, + COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, + :installed_app_store_app_type AS activity_type, + ua.created_at AS created_at, + JSON_OBJECT( + 'host_id', ua.host_id, + 'host_display_name', hdn.display_name, + -- 'software_title', st.name, + -- 'app_store_id', hvsi.adam_id, + 'command_uuid', ua.execution_id, + -- 'self_service', hvsi.self_service IS TRUE, + -- status is always pending because only pending MDM commands are upcoming. + 'status', 'pending_install' + ) AS details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, -- also, cancellable if topmost = 1 + ua.priority as priority + FROM + upcoming_activities ua + LEFT OUTER JOIN + users u ON ua.user_id = u.id + LEFT OUTER JOIN + host_display_names hdn ON hdn.host_id = ua.host_id + -- LEFT OUTER JOIN + -- vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform + -- LEFT OUTER JOIN + -- software_titles st ON st.id = vpa.title_id + WHERE + ua.host_id = :host_id AND + ua.activity_type = 'vpp_app_install' `, - ` -SELECT - hvsi.command_uuid AS uuid, - u.name AS name, - u.id AS user_id, - u.gravatar_url as gravatar_url, - u.email as user_email, - :installed_app_store_app_type AS activity_type, - hvsi.created_at AS created_at, - JSON_OBJECT( - 'host_id', hvsi.host_id, - 'host_display_name', hdn.display_name, - 'software_title', st.name, - 'app_store_id', hvsi.adam_id, - 'command_uuid', hvsi.command_uuid, - 'self_service', hvsi.self_service IS TRUE, - -- status is always pending because only pending MDM commands are upcoming. - 'status', :software_status_install_pending - ) AS details -FROM - host_vpp_software_installs hvsi -INNER JOIN - nano_view_queue nvq ON nvq.command_uuid = hvsi.command_uuid -LEFT OUTER JOIN - users u ON hvsi.user_id = u.id -LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hvsi.host_id -LEFT OUTER JOIN - vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform -LEFT OUTER JOIN - software_titles st ON st.id = vpa.title_id -WHERE - nvq.status IS NULL - AND hvsi.host_id = :host_id -`, } listStmt := ` diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 8aded2f3b9f7..0e685ff83015 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -70,7 +70,7 @@ VALUES JSON_OBJECT( 'sync_request', ?, 'is_internal', ?, - 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 74f37454825c..6871db239743 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -774,7 +774,7 @@ VALUES 'installer_filename', ?, 'version', ?, 'software_title_name', ?, - 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` @@ -918,7 +918,7 @@ VALUES 'installer_filename', '', 'version', 'unknown', 'software_title_name', ?, - 'user', (SELECT JSON_OBJECT('name', name, 'email', email) FROM users WHERE id = ?) + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` From 53033ec52a83c40de2223ce8e3c582a29279cb20 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 14 Jan 2025 09:17:16 -0500 Subject: [PATCH 14/15] Finish list upcoming host activities queries, tests --- server/datastore/mysql/activities.go | 40 ++--- server/datastore/mysql/activities_test.go | 186 ++++++++++++---------- server/service/activities.go | 5 +- 3 files changed, 129 insertions(+), 102 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 22c64f17864e..f8f3495a207a 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -264,13 +264,12 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint // NOTE: Be sure to update both the count (above) and list statements (below) // if the query condition is modified. - // TODO(mna): use the upcoming queue instead (but also include in-execution stuff - // that have no results yet)? listStmts := []string{ // list pending scripts + // TODO(mna): should the user name IF use fleet_initiated? `SELECT ua.execution_id as uuid, - IF(sua.policy_id IS NOT NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) as name, -- TODO: should that use fleet_initiated + IF(sua.policy_id IS NOT NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) as name, u.id as user_id, COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, @@ -281,7 +280,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'host_display_name', COALESCE(hdn.display_name, ''), 'script_name', COALESCE(ses.name, COALESCE(scr.name, '')), 'script_execution_id', ua.execution_id, - 'async', NOT JSON_EXTRACT(ua.payload, '$.sync_request), + 'async', NOT JSON_EXTRACT(ua.payload, '$.sync_request'), 'policy_id', sua.policy_id, 'policy_name', p.name ) as details, @@ -289,7 +288,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint ua.priority as priority FROM upcoming_activities ua - INNER JOIN + INNER JOIN script_upcoming_activities sua ON sua.upcoming_activity_id = ua.id LEFT OUTER JOIN users u ON u.id = ua.user_id @@ -306,12 +305,12 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint ua.activity_type = 'script' `, // list pending software installs + // TODO(mna): should the user name IF use fleet_initiated? `SELECT ua.execution_id as uuid, -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), -- so we mark those as "Fleet" - -- TODO: should we use fleet_initiated instead - IF(ua.user_id IS NULL AND NOT JSON_EXTRACT(ua.payload, '$.self_service), 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, + IF(ua.user_id IS NULL AND NOT JSON_EXTRACT(ua.payload, '$.self_service'), 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, ua.user_id as user_id, COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email, @@ -324,7 +323,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_package', COALESCE(si.filename, JSON_EXTRACT(ua.payload, '$.installer_filename')), 'install_uuid', ua.execution_id, 'status', 'pending_install', - 'self_service', JSON_EXTRACT(ua.payload, '$.self_service) IS TRUE, + 'self_service', JSON_EXTRACT(ua.payload, '$.self_service') IS TRUE, 'policy_id', siua.policy_id, 'policy_name', p.name ) as details, @@ -349,11 +348,11 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint ua.activity_type = 'software_install' `, // list pending software uninstalls + // TODO(mna): should the user name IF use fleet_initiated? `SELECT ua.execution_id as uuid, -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), -- so we mark those as "Fleet" - -- TODO: should we use fleet_initiated instead IF(ua.user_id IS NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name, ua.user_id as user_id, COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url, @@ -436,20 +435,23 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint activity_type, created_at, details - FROM ( ` + strings.Join(listStmts, " UNION ALL ") + ` ) AS upcoming ` - listStmt, args, err = sqlx.Named(listStmt, map[string]any{ - "host_id": hostID, - "ran_script_type": fleet.ActivityTypeRanScript{}.ActivityName(), - "installed_software_type": fleet.ActivityTypeInstalledSoftware{}.ActivityName(), - "uninstalled_software_type": fleet.ActivityTypeUninstalledSoftware{}.ActivityName(), - "installed_app_store_app_type": fleet.ActivityInstalledAppStoreApp{}.ActivityName(), - "max_wait_time": seconds, - "software_status_install_pending": fleet.SoftwareInstallPending, - "software_status_uninstall_pending": fleet.SoftwareUninstallPending, + FROM ( ` + strings.Join(listStmts, " UNION ALL ") + ` ) AS upcoming + ORDER BY topmost DESC, priority DESC, created_at ASC` + + listStmt, args, err := sqlx.Named(listStmt, map[string]any{ + "host_id": hostID, + "ran_script_type": fleet.ActivityTypeRanScript{}.ActivityName(), + "installed_software_type": fleet.ActivityTypeInstalledSoftware{}.ActivityName(), + "uninstalled_software_type": fleet.ActivityTypeUninstalledSoftware{}.ActivityName(), + "installed_app_store_app_type": fleet.ActivityInstalledAppStoreApp{}.ActivityName(), }) if err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "build list query from named args") } + + // the ListOptions supported for this query are limited, only the pagination + // OFFSET and LIMIT can be added, so it's fine to have the ORDER BY already + // in the query before calling this (enforced at the server layer). stmt, args := appendListOptionsWithCursorToSQL(listStmt, args, &opt) var activities []*fleet.Activity diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index 45b87374ba7a..5dc9ca5a05c4 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -365,6 +365,9 @@ func testActivityPaginationMetadata(t *testing.T, ds *Datastore) { } func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { + // TODO(mna): uncomment the commented-out non-pending portions of this test + // once the upcoming queue can run. + noUserCtx := context.Background() u := test.NewUser(t, ds, "user1", "user1@example.com", false) @@ -444,40 +447,43 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) // insert a VPP app - vppCommand1, vppCommand2 := "vpp-command-1", "vpp-command-2" - vppApp := &fleet.VPPApp{ - Name: "vpp_no_team_app_1", VPPAppTeam: fleet.VPPAppTeam{VPPAppID: fleet.VPPAppID{AdamID: "3", Platform: fleet.MacOSPlatform}}, - BundleIdentifier: "b3", - } - _, err = ds.InsertVPPAppWithTeam(ctx, vppApp, nil) - require.NoError(t, err) + // vppCommand1, vppCommand2 := "vpp-command-1", "vpp-command-2" + // vppApp := &fleet.VPPApp{ + // Name: "vpp_no_team_app_1", VPPAppTeam: fleet.VPPAppTeam{VPPAppID: fleet.VPPAppID{AdamID: "3", Platform: fleet.MacOSPlatform}}, + // BundleIdentifier: "b3", + // } + // _, err = ds.InsertVPPAppWithTeam(ctx, vppApp, nil) + // require.NoError(t, err) // install the VPP app on h1 - commander, _ := createMDMAppleCommanderAndStorage(t, ds) - err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", false) - require.NoError(t, err) - err = commander.EnqueueCommand( - ctx, - []string{h1.UUID}, - createRawAppleCmd("InstallApplication", vppCommand1), - ) - require.NoError(t, err) - // install the VPP app on h2, self-service - err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", true) - require.NoError(t, err) - err = commander.EnqueueCommand( - ctx, - []string{h1.UUID}, - createRawAppleCmd("InstallApplication", vppCommand2), - ) - require.NoError(t, err) - - // create a sync script request for h1 that has been pending for > MaxWaitTime, will not show up + // commander, _ := createMDMAppleCommanderAndStorage(t, ds) + // err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", false) + // require.NoError(t, err) + // err = commander.EnqueueCommand( + // ctx, + // []string{h1.UUID}, + // createRawAppleCmd("InstallApplication", vppCommand1), + // ) + // require.NoError(t, err) + // // install the VPP app on h2, self-service + // err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", true) + // require.NoError(t, err) + // err = commander.EnqueueCommand( + // ctx, + // []string{h1.UUID}, + // createRawAppleCmd("InstallApplication", vppCommand2), + // ) + // require.NoError(t, err) + + // create a sync script request for h1 that has been pending for > + // MaxWaitTime, will still show up (sync scripts go through the upcoming + // queue as any script) hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "sync", UserID: &u.ID, SyncRequest: true}) require.NoError(t, err) hSyncExpired := hsr.ExecutionID + t.Log("hSyncExpired", hSyncExpired) ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, "UPDATE host_script_results SET created_at = ? WHERE execution_id = ?", time.Now().Add(-(scripts.MaxServerWaitTime + time.Minute)), hSyncExpired) + _, err := q.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ? WHERE execution_id = ?", time.Now().Add(-(scripts.MaxServerWaitTime + time.Minute)), hSyncExpired) return err }) @@ -485,38 +491,49 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h1A := hsr.ExecutionID + t.Log("h1A", h1A) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr2.ID, ScriptContents: scr2.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h1B := hsr.ExecutionID + t.Log("h1B", h1B) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "C", UserID: &u.ID}) require.NoError(t, err) h1C := hsr.ExecutionID + t.Log("h1C", h1C) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "D"}) require.NoError(t, err) h1D := hsr.ExecutionID + t.Log("h1D", h1D) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "E"}) require.NoError(t, err) h1E := hsr.ExecutionID + t.Log("h1E", h1E) + // create some software installs requests for h1, make some complete - h1FooFailed, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) - require.NoError(t, err) + // h1FooFailed, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) + // require.NoError(t, err) h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID, false, nil) require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: h1.ID, - InstallUUID: h1FooFailed, - PreInstallConditionOutput: ptr.String(""), // pre-install failed - }) - require.NoError(t, err) - h1FooInstalled, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) - require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: h1.ID, - InstallUUID: h1FooInstalled, - PreInstallConditionOutput: ptr.String("ok"), - InstallScriptExitCode: ptr.Int(0), - }) - require.NoError(t, err) + t.Log("h1Bar", h1Bar) + // err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + // HostID: h1.ID, + // InstallUUID: h1FooFailed, + // PreInstallConditionOutput: ptr.String(""), // pre-install failed + // }) + // require.NoError(t, err) + // h1FooInstalled, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) + // require.NoError(t, err) + // err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + // HostID: h1.ID, + // InstallUUID: h1FooInstalled, + // PreInstallConditionOutput: ptr.String("ok"), + // InstallScriptExitCode: ptr.Int(0), + // }) + // require.NoError(t, err) // No user for this one and not Self-service, means it was installed by Fleet policy, err := ds.NewTeamPolicy(ctx, 0, &u.ID, fleet.PolicyPayload{ @@ -525,24 +542,28 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { }) require.NoError(t, err) h1Fleet, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID, false, &policy.ID) + t.Log("h1Fleet", h1Fleet) require.NoError(t, err) // create a single pending request for h2, as well as a non-pending one hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h2A := hsr.ExecutionID - hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptContents: "F", UserID: &u.ID}) - require.NoError(t, err) - _, _, err = ds.SetHostScriptExecutionResult(ctx, - &fleet.HostScriptResultPayload{HostID: h2.ID, ExecutionID: hsr.ExecutionID, Output: "ok", ExitCode: 0}) - require.NoError(t, err) - h2F := hsr.ExecutionID + t.Log("h2A", h2A) + // hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptContents: "F", UserID: &u.ID}) + // require.NoError(t, err) + // _, _, err = ds.SetHostScriptExecutionResult(ctx, + // &fleet.HostScriptResultPayload{HostID: h2.ID, ExecutionID: hsr.ExecutionID, Output: "ok", ExitCode: 0}) + // require.NoError(t, err) + // h2F := hsr.ExecutionID // add a pending software install request for h2 h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID, false, nil) require.NoError(t, err) + t.Log("h2Bar", h2Bar) // No user for this one and Self-service, means it was installed by the end user, so the user_id should be null/nil. h2SelfService, err := ds.InsertSoftwareInstallRequest(noUserCtx, h2.ID, sw1Meta.InstallerID, true, nil) require.NoError(t, err) + t.Log("h2SelfService", h2SelfService) setupExpScript := &fleet.Script{Name: "setup_experience_script", ScriptContents: "setup_experience"} err = ds.SetSetupExperienceScript(ctx, setupExpScript) @@ -552,6 +573,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptContents: "setup_experience", SetupExperienceScriptID: &ses.ID}) require.NoError(t, err) h2SetupExp := hsr.ExecutionID + t.Log("h2SetupExp", h2SetupExp) // create pending install and uninstall requests for h3 that will be deleted _, err = ds.InsertSoftwareInstallRequest(ctx, h3.ID, sw3Meta.InstallerID, false, nil) @@ -563,33 +585,35 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { err = ds.DeleteSoftwareInstaller(ctx, sw3Meta.InstallerID) require.NoError(t, err) - // force-set the order of the created_at timestamps - endTime := SetOrderedCreatedAtTimestamps(t, ds, time.Now(), "host_script_results", "execution_id", h1A, h1B) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooFailed, h1Bar) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooInstalled, h1Fleet) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1Fleet) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2SelfService) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2Bar) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2A, h2F) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_vpp_software_installs", "command_uuid", vppCommand1, vppCommand2) - SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2SetupExp) + // // force-set the order of the created_at timestamps + SetOrderedCreatedAtTimestamps(t, ds, time.Now(), "upcoming_activities", "execution_id", + h1A, h1B /*h1FooFailed,*/, h1Bar, h1C, h1D, h1E /*h1FooInstalled,*/, h1Fleet, h2SelfService, h2Bar, h2A /*h2F, vppCommand1, vppCommand2,*/, h2SetupExp) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooFailed, h1Bar) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooInstalled, h1Fleet) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1Fleet) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2SelfService) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2Bar) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2A, h2F) + // endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_vpp_software_installs", "command_uuid", vppCommand1, vppCommand2) + // SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2SetupExp) execIDsWithUser := map[string]bool{ - h1A: true, - h1B: true, - h1C: true, - h1D: false, - h1E: false, - h2A: true, - h2F: true, + hSyncExpired: true, + h1A: true, + h1B: true, + h1C: true, + h1D: false, + h1E: false, + h2A: true, + //h2F: true, h1Fleet: false, h2SelfService: false, h1Bar: true, h2Bar: true, - vppCommand1: true, - vppCommand2: false, - h2SetupExp: false, + // vppCommand1: true, + // vppCommand2: false, + h2SetupExp: false, } execIDsScriptName := map[string]string{ h1A: scr1.Name, @@ -616,37 +640,37 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1A, h1B}, + wantExecs: []string{hSyncExpired, h1A}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1Bar, h1C}, + wantExecs: []string{h1B, h1Bar}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1D, h1E}, + wantExecs: []string{h1C, h1D}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 3, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1Fleet, vppCommand1}, + wantExecs: []string{h1E, h1Fleet}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, }, { opts: fleet.ListOptions{PerPage: 4}, hostID: h1.ID, - wantExecs: []string{h1A, h1B, h1Bar, h1C}, + wantExecs: []string{hSyncExpired, h1A, h1B, h1Bar}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 4}, hostID: h1.ID, - wantExecs: []string{h1D, h1E, h1Fleet, vppCommand1}, + wantExecs: []string{h1C, h1D, h1E, h1Fleet}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, }, { @@ -658,8 +682,8 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 5}, hostID: h2.ID, - wantExecs: []string{h2SelfService, h2Bar, h2A, vppCommand2, h2SetupExp}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false, TotalResults: 5}, + wantExecs: []string{h2SetupExp, h2SelfService, h2Bar, h2A}, // setup experience is top-priority + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false, TotalResults: 4}, }, { opts: fleet.ListOptions{}, @@ -672,7 +696,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { t.Run(fmt.Sprintf("%v: %#v", c.hostID, c.opts), func(t *testing.T) { // always include metadata c.opts.IncludeMetadata = true - c.opts.OrderKey = "created_at" + c.opts.OrderKey = "" c.opts.OrderDirection = fleet.OrderAscending acts, meta, err := ds.ListHostUpcomingActivities(ctx, c.hostID, c.opts) diff --git a/server/service/activities.go b/server/service/activities.go index 0a555aecd5d4..ed04ee60f78d 100644 --- a/server/service/activities.go +++ b/server/service/activities.go @@ -192,8 +192,9 @@ func (svc *Service) ListHostUpcomingActivities(ctx context.Context, hostID uint, // cursor-based pagination is not supported for upcoming activities opt.After = "" - // custom ordering is not supported, always by date (oldest first) - opt.OrderKey = "created_at" + // custom ordering is not supported, always by upcoming queue order + // (acual order is in the query, not set via ListOptions) + opt.OrderKey = "" opt.OrderDirection = fleet.OrderAscending // no matching query support opt.MatchQuery = "" From f00dae06919ada4781461a80367d6c8749119839 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 14 Jan 2025 09:18:28 -0500 Subject: [PATCH 15/15] Update change file --- changes/23913-upcoming-activities-handle-scripts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changes/23913-upcoming-activities-handle-scripts b/changes/23913-upcoming-activities-handle-scripts index 22c952f1966f..3243574927cf 100644 --- a/changes/23913-upcoming-activities-handle-scripts +++ b/changes/23913-upcoming-activities-handle-scripts @@ -1 +1,3 @@ * Added script execution to the new `upcoming_activities` table. +* Added software installs to the new `upcoming_activities` table. +* Updated the list upcoming activities endpoint to use the new `upcoming_activities` table as source of truth.