Minor refactoring and linting. Fix docstring suggested by tidy-002.

tex2pdf_service/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, arXiv
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tex2pdf_service/bin/compile_submissions.py

@@ -1,7 +1,7 @@
 """
 compile_submissions:
 
-Given a directory of submission tarballs, sends them to tex2pdf API, gets the outcome back and
+Given submissions directory, sends them to tex2pdf API, gets the outcome back and
 writes it out to "outcomes" subdirectory of give submissions directory.
 
 eg:
@@ -21,8 +21,6 @@
 sqlite3 score.db 'select outcome from score where not success' > bad.txt
 
 gives you the all of outcome json files to single file once you run the harvest.
-
-
 """
 
 import os

tex2pdf_service/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "arxiv-tex2pdf"
 version = "0.1.0"
-description = "Compiling TeX to PDF"
+description = "Compiling TeX to PDF, and provide HTTP based service"
 authors = ["arxiv.org"]
 packages = [
     { include = "tex2pdf" }

tex2pdf_service/tex2pdf/__init__.py

@@ -22,11 +22,15 @@
 MAX_TIME_BUDGET: float = float(os.environ.get("MAX_TIME_BUDGET", "595"))
 MAX_LATEX_RUNS: int = int(os.environ.get("MAX_LATEX_RUNS", "5"))
 
+# Log level name may be different depending on the service provider
+LOG_LEVEL_NAME = os.environ.get("LOG_LEVEL_NAME", "severity")
+
 
 class CustomJsonFormatter(JsonFormatter):
-    """Logging formatter to play nice with GCP Logs Explorer"""
+    """Logging formatter to play nice with JSON logger"""
     def __init__(self, *args: list, **kwargs: Any):
-        super().__init__(*args, **kwargs, rename_fields={"levelname": "level", "asctime": "time"})
+        super().__init__(*args, **kwargs,
+                         rename_fields={"levelname": LOG_LEVEL_NAME, "asctime": "time"})
 
     def _perform_rename_log_fields(self, log_record: dict) -> None:
         if "color_message" in log_record:

tex2pdf_service/tex2pdf/fastapi_util.py

@@ -3,19 +3,19 @@
 """
 
 from typing import BinaryIO, TextIO
+from io import BufferedIOBase
 from fastapi import BackgroundTasks
-from google.cloud.storage.blob import BlobReader
 from tex2pdf.service_logger import get_logger
 
 
-def close_stream(it: BlobReader | BinaryIO | TextIO, filename: str, extra: dict) -> None:
+def close_stream(it: BufferedIOBase | BinaryIO | TextIO, filename: str, extra: dict) -> None:
     """Close the stream."""
     it.close()
     get_logger().debug("closed stream for %s", filename, extra=extra)
     pass
 
 
-def closer(it: BlobReader | BinaryIO | TextIO, filename: str, extra: dict) -> BackgroundTasks:
+def closer(it: BufferedIOBase | BinaryIO | TextIO, filename: str, extra: dict) -> BackgroundTasks:
     """Create a background task to close the stream."""
     tasks = BackgroundTasks()
     tasks.add_task(close_stream, it, filename, extra)

tex2pdf_service/tex2pdf/tarball.py

@@ -14,9 +14,12 @@
 
 
 class UnsupportedArchive(Exception):
+    """Submitted archive file extension is not recognized"""
     pass
 
+
 class RemovedSubmission(Exception):
+    """Submitted archive contains the removed.txt and cannot proceed"""
     pass
 
 

tex2pdf_service/tex2pdf/tex2pdf_api.py

@@ -72,7 +72,7 @@ class GzipResponse(StreamingResponse):
 
 @app.get("/", response_class=HTMLResponse)
 def healthcheck() -> str:
-    """Health check endpoint. Pinged by GCP."""
+    """Health check endpoint."""
     return '<h1><a href="./docs/">All good!</a></h1>'
 
 
@@ -95,7 +95,7 @@ async def convert_pdf(incoming: UploadFile,
     filename = incoming.filename if incoming.filename else tempfile.mktemp(prefix="download")
     log_extra = {"source_filename": filename}
     logger = get_logger()
-    logger.info("%s",incoming.filename)
+    logger.info("%s", incoming.filename)
     tag = os.path.basename(filename)
     while True:
         [stem, ext] = os.path.splitext(tag)
@@ -115,7 +115,7 @@ async def convert_pdf(incoming: UploadFile,
                 pass
             pass
         driver = ConverterDriver(tempdir, filename, tag=tag, water=watermark_text,
-                                     max_time_budget=timeout_secs)
+                                 max_time_budget=timeout_secs)
         try:
             _pdf_file = driver.generate_pdf()
         except RemovedSubmission:

tex2pdf_service/tex2pdf/tex_inspection.py

@@ -50,12 +50,10 @@ def __init__(self, in_dir: str | None = None):
             pass
         pass
 
-
     def __bool__(self) -> bool:
         """Return True if 00README.XXX is fetched"""
         return self.readme is not None
 
-
     def fetch_00readme(self, filename: str) -> None:
         """Read and parse 00README.XXX file"""
 
@@ -117,8 +115,8 @@ def find_primary_tex(in_dir: str, zzrm: ZeroZeroReadMe) -> typing.List[str]:
     """
 
     losers = zzrm.ignores | zzrm.includes
-    #loser_re_1 = re.compile(r'\\input\{([^}]+)}')
-    #loser_re_2 = re.compile(r'\\input\s+(.+)')
+    # loser_re_1 = re.compile(r'\\input\{([^}]+)}')
+    # loser_re_2 = re.compile(r'\\input\s+(.+)')
 
     round_0: typing.Set[str] = set()
     round_1: typing.Set[str] = set()
@@ -174,7 +172,7 @@ def find_primary_tex(in_dir: str, zzrm: ZeroZeroReadMe) -> typing.List[str]:
     # If it manages to include multiple tex files with conflicting names, let TEX_FILE_EXTS
     # decide the winner.
     round_3: typing.Set[str] = set()
-    stem_dupes: typing.Dict[str, typing.List[str]]  = {}
+    stem_dupes: typing.Dict[str, typing.List[str]] = {}
     for tex_file in round_2:
         [stem, _ext] = os.path.splitext(tex_file.lower())
         if stem not in stem_dupes:
@@ -191,7 +189,7 @@ def find_primary_tex(in_dir: str, zzrm: ZeroZeroReadMe) -> typing.List[str]:
     round_3s = sorted([tex_file for tex_file in round_3], key=lambda x: x.lower())
 
     return [tex_file for tex_file in zzrm.toplevels if tex_file in round_3s] + \
-        [tex_file for tex_file in round_3s if not tex_file in zzrm.toplevels]
+        [tex_file for tex_file in round_3s if tex_file not in zzrm.toplevels]
 
 
 def is_bib(tex_filename: str) -> bool:
@@ -247,14 +245,14 @@ def find_include_graphics_filename(tex_line: str) -> str | None:
     return find_tex_thing(tex_line, includegraphics_re, ["\\includegraphics"])
 
 
-
 def read_ban_data() -> typing.Any:
     """Read the ban data from the YAML file."""
     ban_list_file = os.path.join(os.path.dirname(__file__), "banned_tex.yaml")
     yaml = YAML()
     with open(ban_list_file, "r", encoding="utf-8") as fd:
         return yaml.load(fd)
 
+
 ban_data = read_ban_data()
 
 
@@ -284,6 +282,7 @@ def decide_ban(condition: dict, target: str) -> bool:
 
     return False
 
+
 def maybe_banned_tex_file(filename: str) -> bool:
     """Is this TeX file blacklisted?"""
     for ban in ban_data["ban_list"]:
@@ -293,6 +292,7 @@ def maybe_banned_tex_file(filename: str) -> bool:
         pass
     return False
 
+
 def is_banned_tex_line(line: str) -> bool:
     """Is this TeX line blacklisted? Use this only when maybe_banned_tex_file() returns True. """
     for ban in ban_data["ban_list"]:
@@ -316,18 +316,18 @@ def is_banned_tex(filename: str, line: str) -> bool:
     return False
 
 
-def is_vanilla_tex_line(line: str)-> bool:
+def is_vanilla_tex_line(line: str) -> bool:
     """Check if the line is a vanilla tex line"""
     return line.find('\\special') >= 0
 
 
-def is_pdftex_line(line: str)-> bool:
+def is_pdftex_line(line: str) -> bool:
     """Check if the line is a pdftex line"""
     # return line.startswith('\\def\\') or line.startswith('\\pageno=')
     return line.startswith('\\pageno=')
 
 
-def is_pdflatex_line(line: str)-> bool:
+def is_pdflatex_line(line: str) -> bool:
     """Check if the line is a pdftex line"""
     # Do not check for \usepackage since etex allows
     # \beginpackages

tex2pdf_service/tex2pdf/tex_patching.py

@@ -1,5 +1,5 @@
 """
-TeX file patching for the GCP PDF generation.
+TeX file patching for the Tex to PDF generation.
 
 The primary motivation of this is, using Pygmentize for code highlighting, the package option
 for first and second run of latex command needs to be changed.

tex2pdf_service/tex2pdf/tex_to_pdf_converters.py

@@ -48,7 +48,12 @@ def __init__(self, conversion_tag: str, zzrm: ZeroZeroReadMe | None = None,
         self.log = ""
         self.log_extra = {ID_TAG: self.conversion_tag}
         self.init_time = time.perf_counter() if init_time is None else init_time
-        self.max_time_budget = float(MAX_TIME_BUDGET) if max_time_budget is None else max_time_budget
+        try:
+            default_max = float(MAX_TIME_BUDGET)
+        except:
+            default_max = 595
+            pass
+        self.max_time_budget = default_max if max_time_budget is None else max_time_budget
         pass
 
     def time_left(self) -> float:
@@ -75,7 +80,7 @@ def produce_pdf(self, tex_file: str, work_dir: str, in_dir: str, out_dir: str) -
         pass
 
     def _exec_cmd(self, args: typing.List[str], child_dir: str, work_dir: str,
-                  extra:dict|None=None) -> typing.Tuple[dict[str, typing.Any], str, str]:
+                  extra: dict | None = None) -> typing.Tuple[dict[str, typing.Any], str, str]:
         """Run the command and return the result"""
         logger = get_logger()
         worker_args = self.decorate_args(args)
@@ -152,7 +157,6 @@ def _report_run(self, run: dict, out: str, err: str, step: str, in_dir: str, out
 
         pass
 
-
     def fetch_log(self, log_file: str) -> None:
         if os.path.exists(log_file):
             with open(log_file, encoding='iso-8859-1') as fd:
@@ -372,6 +376,18 @@ def _base_dvi_to_ps_run(self, stem: str, work_dir: str, in_dir: str, _out_dir: s
         self._report_run(run, out, err, tag, in_dir, work_dir, "ps", ps_filename)
         return run
 
+    def _base_to_dvi_run(self, step: str, stem: str, args: typing.List[str],
+                         work_dir: str, in_dir: str) -> dict:
+        """Runs the given command to generate dvi file and returns the run result."""
+        run, out, err = self._exec_cmd(args, in_dir, work_dir, extra={"step": step})
+        dvi_filename = os.path.join(in_dir, f"{stem}.dvi")
+        self._check_cmd_run(run, dvi_filename)
+        self._report_run(run, out, err, step, in_dir, work_dir, "dvi", dvi_filename)
+        latex_log_file = os.path.join(in_dir, f"{stem}.log")
+        self.fetch_log(latex_log_file)
+        if self.log:
+            run["log"] = self.log
+        return run
 
     def _base_ps_to_pdf_run(self, stem: str, work_dir: str, in_dir: str, out_dir: str) -> dict:
         """Runs ps2pdf command"""
@@ -458,24 +474,13 @@ def produce_pdf(self, tex_file: str, work_dir: str, in_dir: str, out_dir: str) -
         return outcome
 
     def _latex_run(self, tag: str, tex_file: str, work_dir: str, in_dir: str, _out_dir: str) -> dict:
-        stem = self.stem
         # breaks many packages... f"-output-directory=../{bod}"
         args = ["/usr/bin/latex", "-interaction=batchmode", "-file-line-error", "-recorder"]
         if WITH_SHELL_ESCAPE:
             args.append("-shell-escape")
         args.append(tex_file)
-        run, out, err = self._exec_cmd(args, in_dir, work_dir, extra={"step": "latex"})
-        dvi_filename = os.path.join(in_dir, f"{stem}.dvi")
-        self._check_cmd_run(run, dvi_filename)
-        self._report_run(run, out, err, tag, in_dir, work_dir, "dvi", dvi_filename)
-        latex_log_file = os.path.join(in_dir, f"{stem}.log")
-        self.fetch_log(latex_log_file)
-        if self.log:
-            run["log"] = self.log
-        return run
+        return self._base_to_dvi_run(tag, self.stem, args, work_dir, in_dir)
 
-    #def _dvi_to_ps_run(self, work_dir, in_dir, _out_dir, hyperdvi=False) -> dict:
-    #    return self._base_dvi_to_ps_run(self.stem, work_dir, in_dir, _out_dir, hyperdvi=hyperdvi)
 
     def _ps_to_pdf_run(self, work_dir: str, in_dir: str, out_dir: str) -> dict:
         return super()._base_ps_to_pdf_run(self.stem, work_dir, in_dir, out_dir)
@@ -734,8 +739,8 @@ def produce_pdf(self, tex_file: str, work_dir: str, in_dir: str, out_dir: str) -
         self._args = args
 
         # tex run
-        step = "tex_to_ps_run"
-        run = self._tex_run(step, work_dir, in_dir, out_dir)
+        step = "tex_to_dvi_run"
+        run = self._base_to_dvi_run(step, self.stem, args, work_dir, in_dir)
         dvi_size = run["dvi"]["size"]
         if not dvi_size:
             outcome.update({"status": "fail", "step": step,
@@ -760,20 +765,6 @@ def produce_pdf(self, tex_file: str, work_dir: str, in_dir: str, out_dir: str) -
         logger.debug("tex.ps_to_pdf", extra={ID_TAG: self.conversion_tag, "outcome": outcome})
         return outcome
 
-    def _tex_run(self, step: str, work_dir: str, in_dir: str, _out_dir: str) -> dict[str, typing.Any]:
-        """Runs tex command"""
-        args = self._args
-        stem = self.stem
-        run, out, err = self._exec_cmd(args, in_dir, work_dir, extra={"step": step})
-        dvi_filename = os.path.join(in_dir, f"{stem}.dvi")
-        self._check_cmd_run(run, dvi_filename)
-        self._report_run(run, out, err, step, in_dir, work_dir, "dvi", dvi_filename)
-        command_log_file = os.path.join(in_dir, f"{stem}.log")
-        self.fetch_log(command_log_file)
-        if self.log:
-            run["log"] = self.log
-        return run
-
     def _dvi_to_ps_run(self, work_dir: str, in_dir: str, _out_dir: str, hyperdvi: bool=False) -> dict:
         """Run dvips to produce ps."""
         return self._base_dvi_to_ps_run(self.stem, work_dir, in_dir, _out_dir, hyperdvi=hyperdvi)