meta: Add concept of site 'availability'

Uses the same model as Domain.status. Credit goes to @theo-o for the
name (and the original concept of Domain.status that this is based off
of).

Essentially, a site's availability can be one of the three values:
- Enabled: Normal functionality.
- Not served: The site is not served publicly, but otherwise it
  functions normally.
- Disabled: The site is not served publicly, and only superusers can
  view/edit site information through the web interface. (Note that for
  non-superusers who normally have access to the site, the site still
  appears on the home page. However, attempting to visit the "site
  information" page will give a descriptive error message, and visiting
  any other page related to that viewing/editing information regarding
  that site will give a 404.)

The "not served"/"disabled" availabilities are implemented by 1) scaling
the Docker service to 0 replicas and 2) editing the Nginx config to
return an error page.

Note that the large number of changes to the views/consumers was
necessary to enforce the new permission restrictions.

manager/director/apps/sites/consumers.py

@@ -163,6 +163,9 @@ def dump_site_info(self) -> Union[Dict[str, Any], None, bool]:
         except Site.DoesNotExist:
             return None
 
+        if not self.site.can_be_edited_by(self.scope["user"]):
+            return False
+
         if self.site.type != old_type:
             return False
 
@@ -175,6 +178,7 @@ def dump_site_info(self) -> Union[Dict[str, Any], None, bool]:
             "type": self.site.type,
             "type_display": self.site.get_type_display(),
             "users": list(self.site.users.values_list("username", flat=True)),
+            "is_being_served": self.site.is_being_served,
         }
 
         if self.site.has_database:
@@ -259,6 +263,11 @@ async def connect(self) -> None:
             await self.close()
             return
 
+        assert self.site is not None
+        await self.channel_layer.group_add(
+            self.site.channels_group_name, self.channel_name,
+        )
+
         self.connected = True
         await self.accept()
 
@@ -343,6 +352,18 @@ async def mainloop(self) -> None:
             elif isinstance(msg, str):
                 await self.send(text_data=msg)
 
+    async def site_updated(self, event: Dict[str, Any]) -> None:  # pylint: disable=unused-argument
+        if self.site is not None:
+            await database_sync_to_async(self.site.refresh_from_db)()
+
+            if not database_sync_to_async(self.site.can_be_edited_by)(self.scope["user"]):
+                await self.close()
+
+    async def operation_updated(
+        self, event: Dict[str, Any]  # pylint: disable=unused-argument
+    ) -> None:
+        pass
+
     async def disconnect(self, code: int) -> None:  # pylint: disable=unused-argument
         # Clean up
 
@@ -387,6 +408,11 @@ async def connect(self) -> None:
             await self.close()
             return
 
+        assert self.site is not None
+        await self.channel_layer.group_add(
+            self.site.channels_group_name, self.channel_name,
+        )
+
         self.connected = True
         await self.accept()
 
@@ -444,6 +470,18 @@ async def monitor_mainloop(
             elif isinstance(msg, str):
                 await self.send(text_data=msg)
 
+    async def site_updated(self, event: Dict[str, Any]) -> None:  # pylint: disable=unused-argument
+        if self.site is not None:
+            await database_sync_to_async(self.site.refresh_from_db)()
+
+            if not database_sync_to_async(self.site.can_be_edited_by)(self.scope["user"]):
+                await self.close()
+
+    async def operation_updated(
+        self, event: Dict[str, Any]  # pylint: disable=unused-argument
+    ) -> None:
+        pass
+
     async def disconnect(self, code: int) -> None:  # pylint: disable=unused-argument
         self.site = None
         self.connected = False
@@ -487,6 +525,11 @@ async def connect(self) -> None:
             await self.close()
             return
 
+        assert self.site is not None
+        await self.channel_layer.group_add(
+            self.site.channels_group_name, self.channel_name,
+        )
+
         await self.open_log_connection()
 
         if self.logs_websock is None:
@@ -526,6 +569,18 @@ async def mainloop(self) -> None:
             elif isinstance(msg, str):
                 await self.send(text_data=msg)
 
+    async def site_updated(self, event: Dict[str, Any]) -> None:  # pylint: disable=unused-argument
+        if self.site is not None:
+            await database_sync_to_async(self.site.refresh_from_db)()
+
+            if not database_sync_to_async(self.site.can_be_edited_by)(self.scope["user"]):
+                await self.close()
+
+    async def operation_updated(
+        self, event: Dict[str, Any]  # pylint: disable=unused-argument
+    ) -> None:
+        pass
+
     async def disconnect(self, code: int) -> None:  # pylint: disable=unused-argument
         self.site = None
 
@@ -572,10 +627,14 @@ async def connect(self) -> None:
 
         for site_id in self.site_ids:
             try:
-                await get_site_for_user(self.scope["user"], id=site_id)
+                site = await get_site_for_user(self.scope["user"], id=site_id)
             except Site.DoesNotExist:
                 await self.close()
                 return
+            else:
+                await self.channel_layer.group_add(
+                    site.channels_group_name, self.channel_name,
+                )
 
         self.connected = True
         await self.accept()
@@ -620,6 +679,20 @@ async def monitor_mainloop(
             elif isinstance(msg, str):
                 await self.send(text_data=msg)
 
+    async def site_updated(self, event: Dict[str, Any]) -> None:  # pylint: disable=unused-argument
+        for site_id in self.site_ids:
+            site = Site.objects.get(id=site_id)
+
+            await database_sync_to_async(site.refresh_from_db)()
+
+            if not database_sync_to_async(site.can_be_edited_by)(self.scope["user"]):
+                await self.close()
+
+    async def operation_updated(
+        self, event: Dict[str, Any]  # pylint: disable=unused-argument
+    ) -> None:
+        pass
+
     async def disconnect(self, code: int) -> None:  # pylint: disable=unused-argument
         self.site_ids.clear()
         self.connected = False
@@ -630,4 +703,4 @@ async def disconnect(self, code: int) -> None:  # pylint: disable=unused-argumen
 
 @database_sync_to_async
 def get_site_for_user(user, **kwargs: Any) -> Site:
-    return cast(Site, Site.objects.filter_for_user(user).get(**kwargs))
+    return cast(Site, Site.objects.editable_by_user(user).get(**kwargs))

manager/director/apps/sites/forms.py

@@ -276,6 +276,14 @@ class SiteResourceLimitsForm(forms.Form):
     )
 
 
+class SiteAvailabilityForm(forms.Form):
+    availability = forms.ChoiceField(
+        required=False,
+        choices=Site.AVAILABILITIES,
+        widget=forms.Select(attrs={"class": "form-control"}),
+    )
+
+
 class DockerImageForm(forms.ModelForm):
     def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)

manager/director/apps/sites/migrations/0042_auto_20200618_1356.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.13 on 2020-06-18 17:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0041_remove_action_equivalent_command'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='site',
+            name='admin_comments',
+            field=models.TextField(blank=True, help_text='Administrative comments. If availability != enabled, this will be shown to the user.'),
+        ),
+        migrations.AddField(
+            model_name='site',
+            name='availability',
+            field=models.CharField(choices=[('enabled', 'Enabled (fully functional)'), ('not-served', 'Not served publicly'), ('disabled', 'Disabled (not served, only viewable/editable by admins)')], default='enabled', help_text='Controls availability of the site (whether it is served publicly and whether it is editable)', max_length=10),
+        ),
+    ]

manager/director/apps/sites/migrations/0043_auto_20200618_1402.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.13 on 2020-06-18 18:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0042_auto_20200618_1356'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='site',
+            name='admin_comments',
+            field=models.TextField(blank=True, help_text="Administrative comments. All users who have access to the site will always be able to see this, even if the site's 'availability' is 'disabled'."),
+        ),
+    ]

manager/director/apps/sites/models.py

@@ -21,12 +21,31 @@
 
 
 class SiteQuerySet(models.query.QuerySet):
-    def filter_for_user(self, user) -> "models.query.QuerySet[Site]":
+    def listable_by_user(self, user) -> "models.query.QuerySet[Site]":
+        """WARNING: Use editable_by_user() for permission checks instead, unless
+        you immediately check site.can_be_edited_by(user) and handle accordingly.
+
+        The purpose of this function is that if a site has been disabled, the users
+        who have access to it should be able to see that it is still present (i.e.
+        it doesn't "disappear"). However, all other permissions checks for
+        viewing/editing information use `editable_by_user()`, which only allows access
+        if the site is not disabled.
+
+        """
+
         if user.is_superuser:
             return self.all()
         else:
             return self.filter(users=user)
 
+    def editable_by_user(self, user) -> "models.query.QuerySet[Site]":
+        query = self.listable_by_user(user)
+
+        if not user.is_superuser:
+            query = query.filter(availability__in=["enabled", "not-served"])
+
+        return query
+
 
 class Site(models.Model):
     SITE_TYPES = [
@@ -42,6 +61,12 @@ class Site(models.Model):
         ("other", "Other"),
     ]
 
+    AVAILABILITIES = [
+        ("enabled", "Enabled (fully functional)"),
+        ("not-served", "Not served publicly"),
+        ("disabled", "Disabled (not served, only viewable/editable by admins)"),
+    ]
+
     objects: Any = SiteQuerySet.as_manager()
 
     # Website name
@@ -75,9 +100,37 @@ class Site(models.Model):
         "Database", null=True, blank=True, on_delete=models.SET_NULL, related_name="site"
     )
 
+    availability = models.CharField(
+        max_length=max(len(item[0]) for item in AVAILABILITIES),
+        choices=AVAILABILITIES,
+        default="enabled",
+        help_text="Controls availability of the site (whether it is served publicly and whether it "
+        "is editable)",
+    )
+
+    admin_comments = models.TextField(
+        null=False,
+        blank=True,
+        help_text="Administrative comments. All users who have access to the site will always be "
+        "able to see this, even if the site's 'availability' is 'disabled'.",
+    )
+
     # Tell Pylint about the implicit related field
     resource_limits: "SiteResourceLimits"
 
+    def can_be_edited_by(self, user) -> bool:
+        return user.is_authenticated and (
+            user.is_superuser
+            or (
+                self.users.filter(id=user.id).exists()
+                and self.availability in ["enabled", "not-served"]
+            )
+        )
+
+    @property
+    def is_being_served(self) -> bool:
+        return self.availability == "enabled"
+
     def list_urls(self) -> List[str]:
         urls = [
             ("https://" + domain) for domain in self.domain_set.values_list("domain", flat=True)
@@ -112,6 +165,7 @@ def serialize_for_appserver(self) -> Dict[str, Any]:
             "id": self.id,
             "name": self.name,
             "type": self.type,
+            "is_being_served": self.is_being_served,
             "no_redirect_domains": list({split_domain(url) for url in self.list_urls()}),
             "primary_url_base": main_url,
             "database_info": (

manager/director/apps/sites/operations.py

@@ -17,6 +17,7 @@
     regen_site_secrets_task,
     rename_site_task,
     restart_service_task,
+    update_availability_task,
     update_image_task,
     update_resource_limits_task,
 )
@@ -84,6 +85,13 @@ def update_resource_limits(
     send_operation_updated_message(site)
 
 
+def update_availability(site: Site, availability: str) -> None:
+    operation = Operation.objects.create(site=site, type="update_availability")
+    update_availability_task.delay(operation.id, availability)
+
+    send_operation_updated_message(site)
+
+
 def update_image(
     site: Site, base_image_name: str, write_run_sh_file: bool, package_names: List[str],
 ) -> None:

manager/director/apps/sites/tasks.py

@@ -215,6 +215,34 @@ def update_resource_limits_object(
         )
 
 
+@shared_task
+def update_availability_task(operation_id: int, availability: str) -> None:
+    scope: Dict[str, Any] = {
+        "availability": availability,
+    }
+
+    site = Site.objects.get(operation__id=operation_id)
+
+    with auto_run_operation_wrapper(operation_id, scope) as wrapper:
+        wrapper.add_action("Pinging appservers", actions.find_pingable_appservers)
+
+        @wrapper.add_action("Updating availability")
+        def update_availability(
+            site: Site, scope: Dict[str, Any],
+        ) -> Iterator[Union[Tuple[str, str], str]]:
+            yield ("before_state", site.availability)
+            yield ("after_state", scope["availability"])
+            site.availability = scope["availability"]
+            site.save()
+
+        wrapper.add_action(
+            "Updating appserver configuration", actions.update_appserver_nginx_config
+        )
+
+        if site.type == "dynamic":
+            wrapper.add_action("Updating Docker service", actions.update_docker_service)
+
+
 @shared_task
 def regen_site_secrets_task(operation_id: int) -> None:
     scope: Dict[str, Any] = {}

manager/director/apps/sites/urls.py

@@ -67,6 +67,7 @@
     path("restart/raw/", views.sites.restart_raw_view, name="restart_service_raw"),
     # Admin-only
     path("resource-limits/", views.maintenance.resource_limits_view, name="resource_limits"),
+    path("availability/", views.maintenance.availability_view, name="availability"),
     path("edit/", include(edit_patterns)),
     path("database/", include(database_patterns)),
     path("files/", include(file_patterns)),

manager/director/apps/sites/views/database.py

@@ -18,7 +18,7 @@
 @login_required
 @require_accept_guidelines
 def create_database_view(request: HttpRequest, site_id: int) -> HttpResponse:
-    site = get_object_or_404(Site.objects.filter_for_user(request.user), id=site_id)
+    site = get_object_or_404(Site.objects.editable_by_user(request.user), id=site_id)
 
     if site.database is not None:
         return redirect("sites:info", site.id)
@@ -41,7 +41,7 @@ def create_database_view(request: HttpRequest, site_id: int) -> HttpResponse:
 @login_required
 @require_accept_guidelines
 def delete_database_view(request: HttpRequest, site_id: int) -> HttpResponse:
-    site = get_object_or_404(Site.objects.filter_for_user(request.user), id=site_id)
+    site = get_object_or_404(Site.objects.editable_by_user(request.user), id=site_id)
 
     if site.has_operation:
         messages.error(request, "An operation is already being performed on this site")
@@ -58,7 +58,7 @@ def delete_database_view(request: HttpRequest, site_id: int) -> HttpResponse:
 @login_required
 @require_accept_guidelines
 def database_shell_view(request: HttpRequest, site_id: int) -> HttpResponse:
-    site = get_object_or_404(Site.objects.filter_for_user(request.user), id=site_id)
+    site = get_object_or_404(Site.objects.editable_by_user(request.user), id=site_id)
 
     if site.database is None:
         return redirect("sites:info", site.id)