From 67bdb8386ed870af5e67f7677b3fcf0eb4eac3b4 Mon Sep 17 00:00:00 2001
From: Nicolas Hug <nicolashug@fb.com>
Date: Wed, 24 Aug 2022 04:05:22 -0700
Subject: [PATCH] [fbsync] Mark Places365 and SBU download tests as
 xfail(#6389)

Summary:
* disable download test for Places365

* add xfail test for broken download servers

* add more comments

* mark SBU download as failing

Reviewed By: datumbox

Differential Revision: D38824251

fbshipit-source-id: 0244e9377f0e029255eed4ed5c94673d2cfabf5c

Co-authored-by: Nicolas Hug <contact@nicolas-hug.com>
---
 test/test_datasets_download.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/test/test_datasets_download.py b/test/test_datasets_download.py
index 5fa7c6bca44..b4ae5a6d4b7 100644
--- a/test/test_datasets_download.py
+++ b/test/test_datasets_download.py
@@ -433,7 +433,6 @@ def make_parametrize_kwargs(download_configs):
 @pytest.mark.parametrize(
     **make_parametrize_kwargs(
         itertools.chain(
-            places365(),
             caltech101(),
             caltech256(),
             cifar10(),
@@ -448,7 +447,6 @@ def make_parametrize_kwargs(download_configs):
             omniglot(),
             phototour(),
             sbdataset(),
-            sbu(),
             semeion(),
             stl10(),
             svhn(),
@@ -461,6 +459,31 @@ def make_parametrize_kwargs(download_configs):
     )
 )
 def test_url_is_accessible(url, md5):
+    """
+    If you see this test failing, find the offending dataset in the parametrization and move it to
+    ``test_url_is_not_accessible`` and link an issue detailing the problem.
+    """
+    retry(lambda: assert_url_is_accessible(url))
+
+
+@pytest.mark.parametrize(
+    **make_parametrize_kwargs(
+        itertools.chain(
+            places365(),  # https://github.com/pytorch/vision/issues/6268
+            sbu(),  # https://github.com/pytorch/vision/issues/6390
+        )
+    )
+)
+@pytest.mark.xfail
+def test_url_is_not_accessible(url, md5):
+    """
+    As the name implies, this test is the 'inverse' of ``test_url_is_accessible``. Since the download servers are
+    beyond our control, some files might not be accessible for longer stretches of time. Still, we want to know if they
+    come back up, or if we need to remove the download functionality of the dataset for good.
+
+    If you see this test failing, find the offending dataset in the parametrization and move it to
+    ``test_url_is_accessible``.
+    """
     retry(lambda: assert_url_is_accessible(url))