From 2e8711055fd240c4a469083684094712f7a2462e Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 16 Dec 2021 10:19:53 -0500 Subject: [PATCH 1/6] Work around problems building chpl-venv from source i.e. with export CHPL_PIP_INSTALL_PARAMS='--no-binary :all:' --- Signed-off-by: Michael Ferguson --- third-party/chpl-venv/Makefile | 17 +++++++++++------ third-party/chpl-venv/Makefile.include | 4 +++- third-party/chpl-venv/chpldoc-requirements.txt | 2 +- third-party/chpl-venv/chpldoc-requirements1.txt | 1 + third-party/chpl-venv/chpldoc-requirements2.txt | 5 +++++ third-party/chpl-venv/chpldoc-requirements3.txt | 3 +++ 6 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 third-party/chpl-venv/chpldoc-requirements1.txt create mode 100644 third-party/chpl-venv/chpldoc-requirements2.txt create mode 100644 third-party/chpl-venv/chpldoc-requirements3.txt diff --git a/third-party/chpl-venv/Makefile b/third-party/chpl-venv/Makefile index c623a1222d8d..b72c05dc47d8 100644 --- a/third-party/chpl-venv/Makefile +++ b/third-party/chpl-venv/Makefile @@ -33,14 +33,18 @@ $(CHPL_VENV_VIRTUALENV_DIR_OK): $(PYTHON) -m venv $(CHPL_VENV_VIRTUALENV_DIR) export PATH="$(CHPL_VENV_VIRTUALENV_BIN):$$PATH" && \ export VIRTUAL_ENV=$(CHPL_VENV_VIRTUALENV_DIR) && \ - $(PIP) install \ - --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) wheel && \ + $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + wheel && \ + $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE1) && \ + $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2) && \ touch $(CHPL_VENV_VIRTUALENV_DIR_OK) # Phony convenience target for creating virtualenv. create-virtualenv: $(CHPL_VENV_VIRTUALENV_DIR_OK) -$(CHPL_VENV_CHPLDEPS_MAIN): $(CHPL_VENV_VIRTUALENV_DIR_OK) $(CHPL_VENV_TEST_REQUIREMENTS_FILE) $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE) $(CHPL_VENV_C2CHAPEL_REQUIREMENTS_FILE) chpldeps-main.py +$(CHPL_VENV_CHPLDEPS_MAIN): $(CHPL_VENV_VIRTUALENV_DIR_OK) $(CHPL_VENV_TEST_REQUIREMENTS_FILE) $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE1) $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2) $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE3) $(CHPL_VENV_C2CHAPEL_REQUIREMENTS_FILE) chpldeps-main.py @# Install dependencies to $(CHPL_VENV_CHPLDEPS) @# Rely on pip to create the directory export PATH="$(CHPL_VENV_VIRTUALENV_BIN):$$PATH" && \ @@ -48,7 +52,9 @@ $(CHPL_VENV_CHPLDEPS_MAIN): $(CHPL_VENV_VIRTUALENV_DIR_OK) $(CHPL_VENV_TEST_REQU $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ --target $(CHPL_VENV_CHPLDEPS) \ -r $(CHPL_VENV_TEST_REQUIREMENTS_FILE) \ - -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE1) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE3) \ -r $(CHPL_VENV_C2CHAPEL_REQUIREMENTS_FILE) && \ cp chpldeps-main.py $(CHPL_VENV_CHPLDEPS_MAIN) @@ -65,8 +71,7 @@ install-requirements: install-chpldeps $(CHPL_VENV_CHPLSPELL_REQS): $(CHPL_VENV_VIRTUALENV_DIR_OK) $(CHPL_VENV_CHPLSPELL_REQUIREMENTS_FILE) export PATH="$(CHPL_VENV_VIRTUALENV_BIN):$$PATH" && \ export VIRTUAL_ENV=$(CHPL_VENV_VIRTUALENV_DIR) && \ - $(PIP) install \ - --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ -r $(CHPL_VENV_CHPLSPELL_REQUIREMENTS_FILE) && \ touch $(CHPL_VENV_CHPLSPELL_REQS) diff --git a/third-party/chpl-venv/Makefile.include b/third-party/chpl-venv/Makefile.include index c5dbed570009..9ef60a02e159 100644 --- a/third-party/chpl-venv/Makefile.include +++ b/third-party/chpl-venv/Makefile.include @@ -6,7 +6,9 @@ CHPL_VENV_DIR=$(shell cd $(THIRD_PARTY_DIR)/chpl-venv && pwd) CHPL_VENV_TEST_REQUIREMENTS_FILE=$(CHPL_VENV_DIR)/test-requirements.txt -CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE=$(CHPL_VENV_DIR)/chpldoc-requirements.txt +CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE1=$(CHPL_VENV_DIR)/chpldoc-requirements1.txt +CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2=$(CHPL_VENV_DIR)/chpldoc-requirements2.txt +CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE3=$(CHPL_VENV_DIR)/chpldoc-requirements3.txt CHPL_VENV_C2CHAPEL_REQUIREMENTS_FILE=$(CHPL_VENV_DIR)/c2chapel-requirements.txt CHPL_VENV_CHPLSPELL_REQUIREMENTS_FILE=$(CHPL_VENV_DIR)/chplspell-requirements.txt diff --git a/third-party/chpl-venv/chpldoc-requirements.txt b/third-party/chpl-venv/chpldoc-requirements.txt index 4facac5156d6..0361bd80bddb 100644 --- a/third-party/chpl-venv/chpldoc-requirements.txt +++ b/third-party/chpl-venv/chpldoc-requirements.txt @@ -3,7 +3,7 @@ MarkupSafe==2.0.1 Pygments==2.9.0 Sphinx==4.3.2 docutils==0.16.0 -sphinx-rtd-theme==0.5.2 +sphinx-rtd-theme==1.0.0 sphinxcontrib-chapeldomain==0.0.21 babel==2.9.1 breathe==4.31.0 diff --git a/third-party/chpl-venv/chpldoc-requirements1.txt b/third-party/chpl-venv/chpldoc-requirements1.txt new file mode 100644 index 000000000000..c3025d5c0f5f --- /dev/null +++ b/third-party/chpl-venv/chpldoc-requirements1.txt @@ -0,0 +1 @@ +MarkupSafe==2.0.1 diff --git a/third-party/chpl-venv/chpldoc-requirements2.txt b/third-party/chpl-venv/chpldoc-requirements2.txt new file mode 100644 index 000000000000..519cf147c407 --- /dev/null +++ b/third-party/chpl-venv/chpldoc-requirements2.txt @@ -0,0 +1,5 @@ +Jinja2==3.0.1 +Pygments==2.9.0 +Sphinx==4.0.2 +docutils==0.16.0 +babel==2.9.1 diff --git a/third-party/chpl-venv/chpldoc-requirements3.txt b/third-party/chpl-venv/chpldoc-requirements3.txt new file mode 100644 index 000000000000..ba1b71fbf7ba --- /dev/null +++ b/third-party/chpl-venv/chpldoc-requirements3.txt @@ -0,0 +1,3 @@ +sphinx-rtd-theme==1.0.0 +sphinxcontrib-chapeldomain==0.0.20 +breathe==4.30.0 From af29e6e1c5b4e5bcdb5888f1e9bd64060b031647 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Wed, 5 Jan 2022 15:59:31 -0500 Subject: [PATCH 2/6] Add CHPL_PIP_FROM_SOURCE When building pip dependencies from source, we seem to need to 1) install them into the temporary venv 2) install them into the venv a particular order with different pip invocations Both of these are workarounds. To make the normal use case less impacted, add CHPL_PIP_FROM_SOURCE which enables these workarounds so that they do not need to be enabled all the time. --- Signed-off-by: Michael Ferguson --- third-party/chpl-venv/Makefile | 42 +++++++++++++++++++++----- third-party/chpl-venv/Makefile.include | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/third-party/chpl-venv/Makefile b/third-party/chpl-venv/Makefile index b72c05dc47d8..107cddb90f70 100644 --- a/third-party/chpl-venv/Makefile +++ b/third-party/chpl-venv/Makefile @@ -6,6 +6,13 @@ CHPL_MAKE_HOST_TARGET = --host include $(CHPL_MAKE_HOME)/make/Makefile.base include $(THIRD_PARTY_DIR)/chpl-venv/Makefile.include +# CHPL_PIP_INSTALL_PARAMS can be set to adjust the pip arguments, +# but if you want to build from source, set CHPL_PIP_FROM_SOURCE + +ifdef CHPL_PIP_FROM_SOURCE + CHPL_PIP_INSTALL_PARAMS=--no-binary :all: +endif + default: all all: test-venv chpldoc-venv @@ -20,9 +27,7 @@ clobber: FORCE clean OLD_PYTHON_ERROR="python3 version 3.5 or later is required to install chpldoc and start_test dependencies. See https://www.python.org/" -# Create the virtualenv to use during build. -# (to allow for a different path to the system python3 in the future) -$(CHPL_VENV_VIRTUALENV_DIR_OK): +$(CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK): @# First check the python version is OK @if $(PYTHON) -c 'import sys; sys.exit(int(sys.version_info[:2] >= (3, 5)))'; then \ echo $(OLD_PYTHON_ERROR); \ @@ -31,14 +36,35 @@ $(CHPL_VENV_VIRTUALENV_DIR_OK): @# Now create the venv to use to get the dependencies $(PYTHON) -m venv $(CHPL_VENV_VIRTUALENV_DIR) + @# Now install wheel so we can pip install export PATH="$(CHPL_VENV_VIRTUALENV_BIN):$$PATH" && \ export VIRTUAL_ENV=$(CHPL_VENV_VIRTUALENV_DIR) && \ - $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ - wheel && \ - $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + $(PIP) install --upgrade \ + $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) wheel && \ + touch $(CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK) + +ifdef CHPL_PIP_FROM_SOURCE +$(CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK): $(CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK) + @# Now install source dependencies so we can build from source + export PATH="$(CHPL_VENV_VIRTUALENV_BIN):$$PATH" && \ + export VIRTUAL_ENV=$(CHPL_VENV_VIRTUALENV_DIR) && \ + $(PIP) install --upgrade \ + $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE1) && \ - $(PIP) install --upgrade $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ - -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2) && \ + $(PIP) install --upgrade \ + $(CHPL_PIP_INSTALL_PARAMS) $(LOCAL_PIP_FLAGS) \ + -r $(CHPL_VENV_CHPLDOC_REQUIREMENTS_FILE2) && \ + touch $(CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK) + +else +$(CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK): $(CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK) + touch $(CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK) + +endif + +# Create the virtualenv to use during build. +# (to allow for a different path to the system python3 in the future) +$(CHPL_VENV_VIRTUALENV_DIR_OK): $(CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK) $(CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK) touch $(CHPL_VENV_VIRTUALENV_DIR_OK) # Phony convenience target for creating virtualenv. diff --git a/third-party/chpl-venv/Makefile.include b/third-party/chpl-venv/Makefile.include index 9ef60a02e159..aa1121631b60 100644 --- a/third-party/chpl-venv/Makefile.include +++ b/third-party/chpl-venv/Makefile.include @@ -25,6 +25,8 @@ PIP=$(PYTHON) -m pip CHPL_VENV_BUILD=$(CHPL_VENV_DIR)/build CHPL_VENV_VIRTUALENV_DIR=$(CHPL_VENV_BUILD)/build-venv CHPL_VENV_VIRTUALENV_DIR_OK=$(CHPL_VENV_BUILD)/build-venv/ok +CHPL_VENV_VIRTUALENV_DIR_DEPS1_OK=$(CHPL_VENV_BUILD)/build-venv/ok1 +CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK=$(CHPL_VENV_BUILD)/build-venv/ok2 CHPL_VENV_VIRTUALENV_BIN=$(CHPL_VENV_VIRTUALENV_DIR)/bin CHPL_VENV_INSTALL=$(CHPL_VENV_DIR)/install CHPL_VENV_CHPLDEPS=$(CHPL_VENV_INSTALL)/chpldeps From a6f377e6a457d46e7e05ff829767d5eb5200e205 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 6 Jan 2022 16:44:28 -0500 Subject: [PATCH 3/6] withoutOuthor.doc.good change per read-the-docs change --- Signed-off-by: Michael Ferguson --- .../compflags/author/withoutAuthor/withoutAuthor.doc.good | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/chpldoc/compflags/author/withoutAuthor/withoutAuthor.doc.good b/test/chpldoc/compflags/author/withoutAuthor/withoutAuthor.doc.good index 239ef320afa5..09bdf2dfefc3 100644 --- a/test/chpldoc/compflags/author/withoutAuthor/withoutAuthor.doc.good +++ b/test/chpldoc/compflags/author/withoutAuthor/withoutAuthor.doc.good @@ -1 +1 @@ - © Copyright n. +

© Copyright n.

From ab8bd0368a33d6bc3247b7b40694bc0bf944c9ef Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 6 Jan 2022 16:55:49 -0500 Subject: [PATCH 4/6] Port version updates to 2,3 files & remove other --- Signed-off-by: Michael Ferguson --- third-party/chpl-venv/chpldoc-requirements.txt | 9 --------- third-party/chpl-venv/chpldoc-requirements2.txt | 2 +- third-party/chpl-venv/chpldoc-requirements3.txt | 4 ++-- 3 files changed, 3 insertions(+), 12 deletions(-) delete mode 100644 third-party/chpl-venv/chpldoc-requirements.txt diff --git a/third-party/chpl-venv/chpldoc-requirements.txt b/third-party/chpl-venv/chpldoc-requirements.txt deleted file mode 100644 index 0361bd80bddb..000000000000 --- a/third-party/chpl-venv/chpldoc-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Jinja2==3.0.1 -MarkupSafe==2.0.1 -Pygments==2.9.0 -Sphinx==4.3.2 -docutils==0.16.0 -sphinx-rtd-theme==1.0.0 -sphinxcontrib-chapeldomain==0.0.21 -babel==2.9.1 -breathe==4.31.0 diff --git a/third-party/chpl-venv/chpldoc-requirements2.txt b/third-party/chpl-venv/chpldoc-requirements2.txt index 519cf147c407..bf99218cf8d9 100644 --- a/third-party/chpl-venv/chpldoc-requirements2.txt +++ b/third-party/chpl-venv/chpldoc-requirements2.txt @@ -1,5 +1,5 @@ Jinja2==3.0.1 Pygments==2.9.0 -Sphinx==4.0.2 +Sphinx==4.3.2 docutils==0.16.0 babel==2.9.1 diff --git a/third-party/chpl-venv/chpldoc-requirements3.txt b/third-party/chpl-venv/chpldoc-requirements3.txt index ba1b71fbf7ba..9d83860f51b8 100644 --- a/third-party/chpl-venv/chpldoc-requirements3.txt +++ b/third-party/chpl-venv/chpldoc-requirements3.txt @@ -1,3 +1,3 @@ sphinx-rtd-theme==1.0.0 -sphinxcontrib-chapeldomain==0.0.20 -breathe==4.30.0 +sphinxcontrib-chapeldomain==0.0.21 +breathe==4.31.0 From 0863f384eb3abd636de266c7803950937c73f9aa Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Fri, 7 Jan 2022 10:37:37 -0500 Subject: [PATCH 5/6] Add comment --- Signed-off-by: Michael Ferguson --- third-party/chpl-venv/chpldoc-requirements1.txt | 1 + third-party/chpl-venv/chpldoc-requirements2.txt | 1 + third-party/chpl-venv/chpldoc-requirements3.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/third-party/chpl-venv/chpldoc-requirements1.txt b/third-party/chpl-venv/chpldoc-requirements1.txt index c3025d5c0f5f..cf92ecfefc0e 100644 --- a/third-party/chpl-venv/chpldoc-requirements1.txt +++ b/third-party/chpl-venv/chpldoc-requirements1.txt @@ -1 +1,2 @@ +# Split into 3 files to work around problems with CHPL_PIP_FROM_SOURCE MarkupSafe==2.0.1 diff --git a/third-party/chpl-venv/chpldoc-requirements2.txt b/third-party/chpl-venv/chpldoc-requirements2.txt index bf99218cf8d9..aea4b40ac21a 100644 --- a/third-party/chpl-venv/chpldoc-requirements2.txt +++ b/third-party/chpl-venv/chpldoc-requirements2.txt @@ -1,3 +1,4 @@ +# Split into 3 files to work around problems with CHPL_PIP_FROM_SOURCE Jinja2==3.0.1 Pygments==2.9.0 Sphinx==4.3.2 diff --git a/third-party/chpl-venv/chpldoc-requirements3.txt b/third-party/chpl-venv/chpldoc-requirements3.txt index 9d83860f51b8..d4df5d34213c 100644 --- a/third-party/chpl-venv/chpldoc-requirements3.txt +++ b/third-party/chpl-venv/chpldoc-requirements3.txt @@ -1,3 +1,4 @@ +# Split into 3 files to work around problems with CHPL_PIP_FROM_SOURCE sphinx-rtd-theme==1.0.0 sphinxcontrib-chapeldomain==0.0.21 breathe==4.31.0 From cb7bdf64cf18f5a5dad69031d7b79e168fdca2d5 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Fri, 7 Jan 2022 10:58:03 -0500 Subject: [PATCH 6/6] Prediff away OS reason to not make /docs Seeing different errors on different systems - some Macs reporting Read-only file system but other systems report Permission denied. Since this depends on configuration and possibly OS, just leave it out. The test still checks for failure to create the directory. --- Signed-off-by: Michael Ferguson --- test/chpldoc/compflags/folder/PREDIFF | 2 +- test/chpldoc/compflags/folder/failToCreateDir.no-dir.doc.good | 2 +- test/chpldoc/compflags/folder/failToCreateDir.no-perm.doc.good | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/chpldoc/compflags/folder/PREDIFF b/test/chpldoc/compflags/folder/PREDIFF index cc51ec9c99fd..aadfb206963e 100755 --- a/test/chpldoc/compflags/folder/PREDIFF +++ b/test/chpldoc/compflags/folder/PREDIFF @@ -1,4 +1,4 @@ #!/usr/bin/env bash sed -e '/^Running Sphinx v/,/^HTML files are at:/d' $2 > $2.tmp -mv $2.tmp $2 +sed -e 's/due to:.*$/due to: _/g' $2.tmp > $2 diff --git a/test/chpldoc/compflags/folder/failToCreateDir.no-dir.doc.good b/test/chpldoc/compflags/folder/failToCreateDir.no-dir.doc.good index 5576a65772e7..0f3b8aef9dfe 100644 --- a/test/chpldoc/compflags/folder/failToCreateDir.no-dir.doc.good +++ b/test/chpldoc/compflags/folder/failToCreateDir.no-dir.doc.good @@ -1 +1 @@ -error: Failed to create directory: fake-docs-root/deep/dir/docs due to: No such file or directory +error: Failed to create directory: fake-docs-root/deep/dir/docs due to: _ diff --git a/test/chpldoc/compflags/folder/failToCreateDir.no-perm.doc.good b/test/chpldoc/compflags/folder/failToCreateDir.no-perm.doc.good index 3127218cb0ad..adcc9c8125f7 100644 --- a/test/chpldoc/compflags/folder/failToCreateDir.no-perm.doc.good +++ b/test/chpldoc/compflags/folder/failToCreateDir.no-perm.doc.good @@ -1 +1 @@ -error: Failed to create directory: /docs due to: Permission denied +error: Failed to create directory: /docs due to: _