From 82e3ec7d67a284d42df694a2d513545041c5f083 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:09:54 +0100 Subject: [PATCH 01/10] Switch to poetry On branch 382-poetry-docs Changes to be committed: modified: .github/workflows/docs.yml --- .github/workflows/docs.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eb9327ce6..868b15817 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -46,7 +46,7 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} submodules: true fetch-depth: 0 - + - name: Check License Header uses: apache/skywalking-eyes/header@main with: @@ -68,17 +68,24 @@ jobs: 🤖 By [surge-preview](https://surge.sh/) body-include: '' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Poetry + uses: abatilo/actions-poetry@v2 + - name: Install dependencies run: | - sudo apt-get update -y - sudo apt-get install -y ca-certificates cmake doxygen python3-pip - sudo pip3 install -r docs/requirements.txt + cd pyspark + poetry install --with=docs,spark - name: Generate Doc run: | export JAVA_HOME=${JAVA_HOME_11_X64} pushd docs - make html + make html_poetry popd - name: Preview using surge From 8aba92d5089679c1492d69b7b19684610f74ccbe Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:13:59 +0100 Subject: [PATCH 02/10] Add missing deps On branch 382-poetry-docs Changes to be committed: modified: .github/workflows/docs.yml --- .github/workflows/docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 868b15817..39a83c6dd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -78,6 +78,8 @@ jobs: - name: Install dependencies run: | + sudo apt-get update -y + sudo apt-get install -y ca-certificates doxygen cd pyspark poetry install --with=docs,spark From be54cfbd52ac8ec460314729f25673be9ef7f627 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:27:48 +0100 Subject: [PATCH 03/10] Fix Makefile On branch 382-poetry-docs Changes to be committed: modified: docs/Makefile --- docs/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index f8fda06d0..b36222aec 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,8 +6,8 @@ # Do not fail the build if there are warnings SPHINXOPTS = -j8 SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build +SOURCEDIR = docs +BUILDDIR = docs/_build # Internal variables. ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(SOURCEDIR) @@ -61,16 +61,16 @@ pyspark-apidoc: poetry run sphinx-apidoc -o $(ROOTDIR)/docs/pyspark/api graphar_pyspark/ .PHONY: html-poetry -html-poetry: - cd $(ROOTDIR)/pyspark && \ - poetry run bash -c "cd $(ROOTDIR)/docs && $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html" +html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc + cd $(ROOTDIR)/pyspark/ && \ + poetry run $(SPHIXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ mvn --no-transfer-progress -P javadoc javadoc:aggregate \ -Dmaven.antrun.skip=true \ -DskipTests \ - -Djavadoc.output.directory=$(ROOTDIR)/docs/$(BUILDDIR)/html/java/ \ + -Djavadoc.output.directory=$(ROOTDIR)/$(BUILDDIR)/html/java/ \ -Djavadoc.output.destDir=reference \ --quiet @echo From 12468b932d874ccacded84567634bdf8993237c6 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:35:42 +0100 Subject: [PATCH 04/10] CI update On branch 382-poetry-docs Changes to be committed: modified: docs/Makefile --- docs/Makefile | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index b36222aec..6db5fb3f5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -27,7 +27,7 @@ help: .PHONY: clean clean: - rm -rf $(BUILDDIR)/* + rm -rf $(ROOTDIR)/$(BUILDDIR)/* .PHONY: cpp-apidoc cpp-apidoc: @@ -40,21 +40,6 @@ spark-apidoc: mvn --no-transfer-progress clean install -DskipTests -Dspotless.check.skip=true && \ mvn --no-transfer-progress scala:doc -.PHONY: html -html: cpp-apidoc spark-apidoc - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - rm -fr $(BUILDDIR)/html/spark/reference - cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ - cd $(ROOTDIR)/java && \ - mvn --no-transfer-progress -P javadoc javadoc:aggregate \ - -Dmaven.antrun.skip=true \ - -DskipTests \ - -Djavadoc.output.directory=$(ROOTDIR)/docs/$(BUILDDIR)/html/java/ \ - -Djavadoc.output.destDir=reference \ - --quiet - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - .PHONY: pyspark-apidoc pyspark-apidoc: cd $(ROOTDIR)/pyspark && \ @@ -65,7 +50,7 @@ html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc cd $(ROOTDIR)/pyspark/ && \ poetry run $(SPHIXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference - cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ + cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(ROOTDIR)/$(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ mvn --no-transfer-progress -P javadoc javadoc:aggregate \ -Dmaven.antrun.skip=true \ From b8a8fd002fb8d99df1296a627db7971ddc548049 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:36:48 +0100 Subject: [PATCH 05/10] Fix typo On branch 382-poetry-docs Changes to be committed: modified: docs/Makefile --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 6db5fb3f5..3750d0acf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -48,7 +48,7 @@ pyspark-apidoc: .PHONY: html-poetry html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc cd $(ROOTDIR)/pyspark/ && \ - poetry run $(SPHIXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + poetry run $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(ROOTDIR)/$(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ From 49672747c91021ff66abd479ef3e4aed4e78cea3 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Tue, 27 Feb 2024 23:58:38 +0100 Subject: [PATCH 06/10] Fix docs On branch 382-poetry-docs Changes to be committed: modified: docs/Makefile --- docs/Makefile | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 3750d0acf..53b2772c3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,11 +6,11 @@ # Do not fail the build if there are warnings SPHINXOPTS = -j8 SPHINXBUILD ?= sphinx-build -SOURCEDIR = docs -BUILDDIR = docs/_build +SOURCEDIR = . +BUILDDIR = _build # Internal variables. -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(SOURCEDIR) +ALLSPHINXOPTS = -d ../docs/$(BUILDDIR)/doctrees $(SPHINXOPTS) DOXYGEN = doxygen ROOTDIR = .. @@ -27,7 +27,7 @@ help: .PHONY: clean clean: - rm -rf $(ROOTDIR)/$(BUILDDIR)/* + rm -rf $(BUILDDIR)/* .PHONY: cpp-apidoc cpp-apidoc: @@ -42,20 +42,19 @@ spark-apidoc: .PHONY: pyspark-apidoc pyspark-apidoc: - cd $(ROOTDIR)/pyspark && \ - poetry run sphinx-apidoc -o $(ROOTDIR)/docs/pyspark/api graphar_pyspark/ + cd ../pyspark && poetry run sphinx-apidoc -o ../docs/pyspark/api graphar_pyspark/ .PHONY: html-poetry -html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc - cd $(ROOTDIR)/pyspark/ && \ - poetry run $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html +html-poetry: spark-apidoc pyspark-apidoc + cd ../pyspark && \ + poetry run $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) ../docs ../docs/$(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference - cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(ROOTDIR)/$(BUILDDIR)/html/spark/reference/ + cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ mvn --no-transfer-progress -P javadoc javadoc:aggregate \ -Dmaven.antrun.skip=true \ -DskipTests \ - -Djavadoc.output.directory=$(ROOTDIR)/$(BUILDDIR)/html/java/ \ + -Djavadoc.output.directory=$(ROOTDIR)/docs/$(BUILDDIR)/html/java/ \ -Djavadoc.output.destDir=reference \ --quiet @echo From 5c447866540779b1cc7a62e114917792f719079a Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Wed, 28 Feb 2024 00:07:05 +0100 Subject: [PATCH 07/10] Try to fix --- docs/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 53b2772c3..1379397c7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,12 +5,12 @@ # from the environment for the first two. # Do not fail the build if there are warnings SPHINXOPTS = -j8 -SPHINXBUILD ?= sphinx-build +SPHINXBUILD ?= sphinx.cmd.build SOURCEDIR = . BUILDDIR = _build # Internal variables. -ALLSPHINXOPTS = -d ../docs/$(BUILDDIR)/doctrees $(SPHINXOPTS) +ALLSPHINXOPTS = -b html -d ../docs/$(BUILDDIR)/doctrees $(SPHINXOPTS) DOXYGEN = doxygen ROOTDIR = .. @@ -45,9 +45,9 @@ pyspark-apidoc: cd ../pyspark && poetry run sphinx-apidoc -o ../docs/pyspark/api graphar_pyspark/ .PHONY: html-poetry -html-poetry: spark-apidoc pyspark-apidoc - cd ../pyspark && \ - poetry run $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) ../docs ../docs/$(BUILDDIR)/html +html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc + cd $(ROOTDIR)/pyspark && \ + poetry run python -m $(SPHINXBUILD) $(ALLSPHINXOPTS) $(ROOTDIR)/docs $(ROOTDIR)/docs/$(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ From d6510c25a64d381e2c1757b2f27763d097ee1974 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Wed, 28 Feb 2024 00:15:07 +0100 Subject: [PATCH 08/10] Try to fix CI --- .github/workflows/docs.yml | 7 +++---- docs/Makefile | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 39a83c6dd..a055a34eb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -80,15 +80,14 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y ca-certificates doxygen - cd pyspark poetry install --with=docs,spark + working-directory: ./pyspark - name: Generate Doc run: | export JAVA_HOME=${JAVA_HOME_11_X64} - pushd docs - make html_poetry - popd + make html-poetry + working-directory: ./docs - name: Preview using surge if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphAr' }} diff --git a/docs/Makefile b/docs/Makefile index 1379397c7..749f261fc 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -46,8 +46,8 @@ pyspark-apidoc: .PHONY: html-poetry html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc - cd $(ROOTDIR)/pyspark && \ - poetry run python -m $(SPHINXBUILD) $(ALLSPHINXOPTS) $(ROOTDIR)/docs $(ROOTDIR)/docs/$(BUILDDIR)/html + echo "Generate docs..." + cd $(ROOTDIR)/pyspark && poetry run python -m $(SPHINXBUILD) $(ALLSPHINXOPTS) $(ROOTDIR)/docs $(ROOTDIR)/docs/$(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference cp -fr $(ROOTDIR)/spark/graphar/target/site/scaladocs $(BUILDDIR)/html/spark/reference/ cd $(ROOTDIR)/java && \ From f8e2ca9fa5859099b3daa86e330be21551ed5e3e Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Wed, 28 Feb 2024 08:39:53 +0100 Subject: [PATCH 09/10] Update from comments On branch 382-poetry-docs Changes to be committed: modified: .github/workflows/docs.yml modified: docs/Makefile deleted: docs/requirements.txt --- .github/workflows/docs.yml | 2 +- docs/Makefile | 2 +- docs/requirements.txt | 11 ----------- 3 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a055a34eb..4a0b154ac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -86,7 +86,7 @@ jobs: - name: Generate Doc run: | export JAVA_HOME=${JAVA_HOME_11_X64} - make html-poetry + make html working-directory: ./docs - name: Preview using surge diff --git a/docs/Makefile b/docs/Makefile index 749f261fc..b9278c0d2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -44,7 +44,7 @@ spark-apidoc: pyspark-apidoc: cd ../pyspark && poetry run sphinx-apidoc -o ../docs/pyspark/api graphar_pyspark/ -.PHONY: html-poetry +.PHONY: html html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc echo "Generate docs..." cd $(ROOTDIR)/pyspark && poetry run python -m $(SPHINXBUILD) $(ALLSPHINXOPTS) $(ROOTDIR)/docs $(ROOTDIR)/docs/$(BUILDDIR)/html diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 8e4deda84..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -breathe -docutils -furo # sphinx theme -nbsphinx -sphinx>=3.0.2 -jinja2>=3.1.2 -sphinx-copybutton -sphinx-panels -sphinxemoji -sphinxext-opengraph -markupsafe==2.0.1 From 401e08ca20b0459f3b8d0940535188de6eb5516a Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Wed, 28 Feb 2024 11:43:39 +0100 Subject: [PATCH 10/10] Updates from comments On branch 382-poetry-docs Changes to be committed: modified: CONTRIBUTING.rst modified: docs/Makefile --- CONTRIBUTING.rst | 11 ++++++++++- docs/Makefile | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1835820e5..122e62398 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -128,7 +128,16 @@ How to generate the document If you want to improve the document, you need to know how to generate the docs. -The documentation is generated using Doxygen and sphinx. You can build GraphAr's documentation in the :code:`docs/` directory using: +The documentation is generated using Doxygen and sphinx. For sphinx and other Python dependency the GraphAr pyspark subproject is used. You need to `install Poetry packaging system first `_. The next step will be installation of docs-related Python dependencies: + +.. code:: shell + $ cd pyspark + $ poetry install --with=docs + +.. note:: + If you want to update PySpark project API documentation, you need also install dependencies from spark group in poetry: poetry install --with=docs,spark. For Cpp API documentation you need to `install also doxygen `_ (in most of POSIX-compatible systems installation is also possible via system package manager). For Java/Scala API documentation you need to `install Maven `_ (this step also may be done via system package manager on the most of POSIX-compatible systems). + +After that you can build GraphAr's documentation in the :code:`docs/` directory using: .. code:: shell diff --git a/docs/Makefile b/docs/Makefile index b9278c0d2..8fc75876d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -45,7 +45,7 @@ pyspark-apidoc: cd ../pyspark && poetry run sphinx-apidoc -o ../docs/pyspark/api graphar_pyspark/ .PHONY: html -html-poetry: cpp-apidoc spark-apidoc pyspark-apidoc +html: cpp-apidoc spark-apidoc pyspark-apidoc echo "Generate docs..." cd $(ROOTDIR)/pyspark && poetry run python -m $(SPHINXBUILD) $(ALLSPHINXOPTS) $(ROOTDIR)/docs $(ROOTDIR)/docs/$(BUILDDIR)/html rm -fr $(BUILDDIR)/html/spark/reference