Build: use environment variable $READTHEDOCS_OUTPUT to define output directory

readthedocs/doc_builder/backends/mkdocs.py

@@ -296,17 +296,8 @@ def build(self):
             'mkdocs',
             self.builder,
             "--clean",
-            # ``site_dir`` is relative to where the mkdocs.yaml file is
-            # https://www.mkdocs.org/user-guide/configuration/#site_dir
-            # Example:
-            #
-            #  when ``--config-file=docs/mkdocs.yml``,
-            # it must be ``--site-dir=../_readthedocs/html``
             "--site-dir",
-            os.path.join(
-                os.path.relpath(self.project_path, os.path.dirname(self.yaml_file)),
-                self.build_dir,
-            ),
+            os.path.join("$READTHEDOCS_OUTPUT", "html"),
             "--config-file",
             os.path.relpath(self.yaml_file, self.project_path),
         ]

readthedocs/doc_builder/backends/sphinx.py

@@ -42,15 +42,15 @@ class BaseSphinx(BaseBuilder):
     # an output file, the parsed source files are cached as "doctree pickles".
     sphinx_doctrees_dir = "_build/doctrees"
 
-    # Output directory relative to where the repository was cloned
-    # (e.g. "_readthedocs/<format>")
+    # Output directory relative to $READTHEDOCS_OUTPUT
+    # (e.g. "html", "htmlzip" or "pdf")
     relative_output_dir = None
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.config_file = self.config.sphinx.configuration
-        self.absolute_output_dir = os.path.abspath(
-            os.path.join(self.project_path, self.relative_output_dir)
+        self.absolute_output_dir = os.path.join(
+            "$READTHEDOCS_OUTPUT", self.relative_output_dir
         )
         try:
             if not self.config_file:
@@ -309,9 +309,7 @@ def build(self):
                 # https://github.com/readthedocs/readthedocs.org/pull/9888#issuecomment-1384649346
                 ".",
                 # Sphinx's output build directory (OUTPUTDIR)
-                os.path.relpath(
-                    self.absolute_output_dir, os.path.dirname(self.config_file)
-                ),
+                self.absolute_output_dir,
             ]
         )
         cmd_ret = self.run(
@@ -368,9 +366,22 @@ def venv_sphinx_supports_latexmk(self):
         )
         return cmd_ret.exit_code == 0
 
+    def _get_absolute_host_output_dir(self):
+        # NOTE: we cannot use `$READTHEDOCS_OUTPUT` environment variable here
+        # because it's not defined in the host. So, we have to re-calculate its
+        # value. We will remove this limitation when we execute the whole
+        # building from inside the Docker container (instead behing a hybrid as it is now)
+        readthedocs_output = os.path.join(
+            self.project.checkout_path(self.version.slug), "_readthedocs/"
+        )
+        absolute_host_output_dir = os.path.join(
+            readthedocs_output, self.relative_output_dir
+        )
+        return absolute_host_output_dir
+
 
 class HtmlBuilder(BaseSphinx):
-    relative_output_dir = "_readthedocs/html"
+    relative_output_dir = "html"
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -399,18 +410,16 @@ def __init__(self, *args, **kwargs):
 
 class LocalMediaBuilder(BaseSphinx):
     sphinx_builder = 'readthedocssinglehtmllocalmedia'
-    relative_output_dir = "_readthedocs/htmlzip"
+    relative_output_dir = "htmlzip"
 
     def _post_build(self):
         """Internal post build to create the ZIP file from the HTML output."""
-        target_file = os.path.abspath(
-            os.path.join(
-                self.absolute_output_dir,
-                # TODO: shouldn't this name include the name of the version as well?
-                # It seems we were using the project's slug previously.
-                # So, keeping it like that for now until we decide make that adjustment.
-                f"{self.project.slug}.zip",
-            )
+        target_file = os.path.join(
+            self.absolute_output_dir,
+            # TODO: shouldn't this name include the name of the version as well?
+            # It seems we were using the project's slug previously.
+            # So, keeping it like that for now until we decide make that adjustment.
+            f"{self.project.slug}.zip",
         )
 
         # **SECURITY CRITICAL: Advisory GHSA-hqwg-gjqw-h5wg**
@@ -422,15 +431,15 @@ def _post_build(self):
         dirname = f"{self.project.slug}-{self.version.slug}"
         self.run(
             "mv",
-            self.relative_output_dir,
+            self.absolute_output_dir,
             str(tmp_dir / dirname),
             cwd=self.project_path,
             record=False,
         )
         self.run(
             "mkdir",
             "--parents",
-            self.relative_output_dir,
+            self.absolute_output_dir,
             cwd=self.project_path,
             record=False,
         )
@@ -448,7 +457,7 @@ def _post_build(self):
 class EpubBuilder(BaseSphinx):
 
     sphinx_builder = "epub"
-    relative_output_dir = "_readthedocs/epub"
+    relative_output_dir = "epub"
 
     def _post_build(self):
         """Internal post build to cleanup EPUB output directory and leave only one .epub file."""
@@ -458,7 +467,9 @@ def _post_build(self):
             f"{self.project.slug}.epub",
         )
 
-        epub_sphinx_filepaths = glob(os.path.join(self.absolute_output_dir, "*.epub"))
+        epub_sphinx_filepaths = glob(
+            os.path.join(self._get_absolute_host_output_dir(), "*.epub")
+        )
         if epub_sphinx_filepaths:
             # NOTE: we currently support only one .epub per version
             epub_filepath = epub_sphinx_filepaths[0]
@@ -469,14 +480,14 @@ def _post_build(self):
             self.run(
                 "rm",
                 "--recursive",
-                self.relative_output_dir,
+                self.absolute_output_dir,
                 cwd=self.project_path,
                 record=False,
             )
             self.run(
                 "mkdir",
                 "--parents",
-                self.relative_output_dir,
+                self.absolute_output_dir,
                 cwd=self.project_path,
                 record=False,
             )
@@ -513,7 +524,7 @@ class PdfBuilder(BaseSphinx):
 
     """Builder to generate PDF documentation."""
 
-    relative_output_dir = "_readthedocs/pdf"
+    relative_output_dir = "pdf"
     sphinx_builder = "latex"
     pdf_file_name = None
 
@@ -536,14 +547,12 @@ def build(self):
             # https://github.com/readthedocs/readthedocs.org/pull/9888#issuecomment-1384649346
             ".",
             # Sphinx's output build directory (OUTPUTDIR)
-            os.path.relpath(
-                self.absolute_output_dir, os.path.dirname(self.config_file)
-            ),
+            self.absolute_output_dir,
             cwd=os.path.dirname(self.config_file),
             bin_path=self.python_env.venv_bin(),
         )
 
-        tex_files = glob(os.path.join(self.absolute_output_dir, "*.tex"))
+        tex_files = glob(os.path.join(self._get_absolute_host_output_dir(), "*.tex"))
         if not tex_files:
             raise BuildUserError("No TeX files were found.")
 
@@ -563,7 +572,9 @@ def _build_latexmk(self, cwd):
         # https://github.com/sphinx-doc/sphinx/blob/master/sphinx/texinputs/Makefile_t
         images = []
         for extension in ("png", "gif", "jpg", "jpeg"):
-            images.extend(Path(self.absolute_output_dir).glob(f"*.{extension}"))
+            images.extend(
+                Path(self._get_absolute_host_output_dir()).glob(f"*.{extension}")
+            )
 
         # FIXME: instead of checking by language here, what we want to check if
         # ``latex_engine`` is ``platex``
@@ -573,13 +584,13 @@ def _build_latexmk(self, cwd):
             # step. I don't know exactly why but most of the documentation that
             # I read differentiate this language from the others. I suppose
             # it's because it mix kanji (Chinese) with its own symbols.
-            pdfs = Path(self.absolute_output_dir).glob("*.pdf")
+            pdfs = Path(self._get_absolute_host_output_dir()).glob("*.pdf")
 
         for image in itertools.chain(images, pdfs):
             self.run(
                 'extractbb',
                 image.name,
-                cwd=self.absolute_output_dir,
+                cwd=self._get_absolute_host_output_dir(),
                 record=False,
             )
 
@@ -590,7 +601,7 @@ def _build_latexmk(self, cwd):
         self.run(
             'cat',
             rcfile,
-            cwd=self.absolute_output_dir,
+            cwd=self._get_absolute_host_output_dir(),
         )
 
         if self.build_env.command_class == DockerBuildCommand:
@@ -618,7 +629,7 @@ def _build_latexmk(self, cwd):
             cls=latex_class,
             cmd=cmd,
             warn_only=True,
-            cwd=self.absolute_output_dir,
+            cwd=self._get_absolute_host_output_dir(),
         )
 
         self.pdf_file_name = f'{self.project.slug}.pdf'
@@ -653,23 +664,23 @@ def _build_pdflatex(self, tex_files):
             cmd_ret = self.build_env.run_command_class(
                 cls=latex_class,
                 cmd=cmd,
-                cwd=self.absolute_output_dir,
+                cwd=self._get_absolute_host_output_dir(),
                 warn_only=True,
             )
             pdf_commands.append(cmd_ret)
         for cmd in makeindex_cmds:
             cmd_ret = self.build_env.run_command_class(
                 cls=latex_class,
                 cmd=cmd,
-                cwd=self.absolute_output_dir,
+                cwd=self._get_absolute_host_output_dir(),
                 warn_only=True,
             )
             pdf_commands.append(cmd_ret)
         for cmd in pdflatex_cmds:
             cmd_ret = self.build_env.run_command_class(
                 cls=latex_class,
                 cmd=cmd,
-                cwd=self.absolute_output_dir,
+                cwd=self._get_absolute_host_output_dir(),
                 warn_only=True,
             )
             pdf_match = PDF_RE.search(cmd_ret.output)
@@ -693,7 +704,10 @@ def _post_build(self):
 
         # NOTE: we currently support only one .pdf per version
         pdf_sphinx_filepath = os.path.join(self.absolute_output_dir, self.pdf_file_name)
-        if os.path.exists(pdf_sphinx_filepath):
+        pdf_sphinx_filepath_host = os.path.join(
+            self._get_absolute_host_output_dir(), self.pdf_file_name
+        )
+        if os.path.exists(pdf_sphinx_filepath_host):
             self.run(
                 "mv",
                 pdf_sphinx_filepath,
@@ -704,14 +718,14 @@ def _post_build(self):
             self.run(
                 "rm",
                 "-r",
-                self.relative_output_dir,
+                self.absolute_output_dir,
                 cwd=self.project_path,
                 record=False,
             )
             self.run(
                 "mkdir",
                 "-p",
-                self.relative_output_dir,
+                self.absolute_output_dir,
                 cwd=self.project_path,
                 record=False,
             )

readthedocs/doc_builder/director.py

@@ -543,6 +543,9 @@ def get_rtd_env_vars(self):
             "READTHEDOCS_VERSION_NAME": self.data.version.verbose_name,
             "READTHEDOCS_PROJECT": self.data.project.slug,
             "READTHEDOCS_LANGUAGE": self.data.project.language,
+            "READTHEDOCS_OUTPUT": os.path.join(
+                self.data.project.checkout_path(self.data.version.slug), "_readthedocs/"
+            ),
         }
         return env
 

readthedocs/doc_builder/environments.py

@@ -130,8 +130,6 @@ def __str__(self):
     # commands, which is not supported anymore
     def run(self):
         """Set up subprocess and execute command."""
-        log.info("Running build command.", command=self.get_command(), cwd=self.cwd)
-
         self.start_time = datetime.utcnow()
         environment = self._environment.copy()
         if 'DJANGO_SETTINGS_MODULE' in environment:
@@ -145,6 +143,13 @@ def run(self):
             env_paths.insert(0, self.bin_path)
         environment['PATH'] = ':'.join(env_paths)
 
+        log.info(
+            "Running build command.",
+            command=self.get_command(),
+            cwd=self.cwd,
+            environment=environment,
+        )
+
         try:
             # When using ``shell=True`` the command should be flatten
             command = self.command