diff --git a/cmd/containerd-shim-runhcs-v1/service_internal.go b/cmd/containerd-shim-runhcs-v1/service_internal.go
index eb85f69722..ad53b03512 100644
--- a/cmd/containerd-shim-runhcs-v1/service_internal.go
+++ b/cmd/containerd-shim-runhcs-v1/service_internal.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
 
 	runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/Microsoft/hcsshim/internal/oci"
@@ -215,7 +216,14 @@ func (s *service) deleteInternal(ctx context.Context, req *task.DeleteRequest) (
 	if err != nil {
 		return nil, err
 	}
-	// TODO: We should be removing the task after this right?
+
+	if req.ExecID == "" {
+		// We've successfully deleted the init exec. Compute resources
+		// should be closed, layers umounted and resources released.
+		// This task should now be done.
+		s.taskOrPod = atomic.Value{}
+	}
+
 	return &task.DeleteResponse{
 		Pid:        uint32(pid),
 		ExitStatus: exitStatus,
diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go
index 7a59e43696..b0243e73d8 100644
--- a/cmd/containerd-shim-runhcs-v1/task_hcs.go
+++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go
@@ -536,8 +536,9 @@ func (ht *hcsTask) KillExec(ctx context.Context, eid string, signal uint32, all
 				}).Warn("failed to kill exec in task")
 			}
 
-			// iterate all
-			return false
+			// Iterate all. Returning false stops the iteration. See:
+			// https://pkg.go.dev/sync#Map.Range
+			return true
 		})
 	}
 	if signal == 0x9 && eid == "" && ht.host != nil {
@@ -578,8 +579,8 @@ func (ht *hcsTask) DeleteExec(ctx context.Context, eid string) (int, uint32, tim
 				ex.ForceExit(ctx, 1)
 			}
 
-			// iterate next. Returning false here, means only sending
-			// Kill to the first task in the map.
+			// Iterate all. Returning false stops the iteration. See:
+			// https://pkg.go.dev/sync#Map.Range
 			return true
 		})
 	}
@@ -639,8 +640,9 @@ func (ht *hcsTask) Pids(ctx context.Context) ([]runhcsopts.ProcessDetails, error
 		ex := value.(shimExec)
 		pidMap[ex.Pid()] = ex.ID()
 
-		// Iterate all
-		return false
+		// Iterate all. Returning false stops the iteration. See:
+		// https://pkg.go.dev/sync#Map.Range
+		return true
 	})
 	pidMap[ht.init.Pid()] = ht.init.ID()
 
@@ -721,8 +723,9 @@ func (ht *hcsTask) waitForHostExit() {
 		ex := value.(shimExec)
 		ex.ForceExit(ctx, 1)
 
-		// iterate all
-		return false
+		// Iterate all. Returning false stops the iteration. See:
+		// https://pkg.go.dev/sync#Map.Range
+		return true
 	})
 	ht.init.ForceExit(ctx, 1)
 	ht.closeHost(ctx)