From 9cdead1aaca91f0811f3ea75dda3ce1a4e0f12a6 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Fri, 24 Sep 2021 12:14:03 +0200 Subject: [PATCH 01/11] Create v8 upgrade workflow --- .github/workflows/v8upgrade.yml | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/v8upgrade.yml diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml new file mode 100644 index 000000000..b1de2596f --- /dev/null +++ b/.github/workflows/v8upgrade.yml @@ -0,0 +1,46 @@ +name: V8 Upgrade + +on: workflow_dispatch + +jobs: + upgrade: + name: Upgrade V8 + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + - name: Update depot_tools fetch config + run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + shell: bash + - name: Add depot_tools to PATH + run: echo "$PWD/deps/depot_tools" >> $GITHUB_PATH + shell: bash + - name: Fetch V8 + run: cd deps && fetch v8 || true + - name: Get latest stable commit ref of v8 + id: v8_latest_commit + run: echo ::set-output name=VALUE::$(curl -sS 'https://omahaproxy.appspot.com/all.json?os=linux&channel=stable' | jq '.[0]["versions"][0]["v8_commit"]') + - name: Get latest stable version number of v8 + id: v8_latest_version + run: echo ::set-output name=VALUE::$(curl -sS 'https://omahaproxy.appspot.com/all.json?os=linux&channel=stable' | jq '.[0]["versions"][0]["v8_version"]') + - name: Checkout v8 to stable commit ref + run: cd deps/v8 && git fetch && git checkout ${{steps.v8_latest_commit.outputs.VALUE}} + - name: Copy deps/v8/include into deps/include + run: cp -r deps/v8/include/* deps/include/ + - name: Create PR metadata + id: pr_metadata + run: | + echo ::set-output name=pr_branch::"v8_${{steps.v8_latest_version.outputs.VALUE}}_upgrade" + - name: Create PR + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Upgrade V8 binaries + branch: ${{steps.pr_metadata.outputs.pr_branch}} + delete-branch: true + title: Upgrade V8 binary + body: Auto-generated pull request to upgarde V8 binary + + From c84d90c91093769dd3d8895452133d5bbbbb1267 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Mon, 27 Sep 2021 12:46:08 +0200 Subject: [PATCH 02/11] Work related to https://github.com/rogchap/v8go/issues/165 - Created an scheduled workflow that runs every month, which automated the steps to upgrade the v8 binaries from source https://github.com/rogchap/v8go#upgrading-the-v8-binaries - Created a new workflow that is triggered by a pull_request which build the v8 binaries for MacOs and Linux. Window is excluded until we figured out a way to solve the compiler issues. --- .github/workflows/v8build_not_for_windows.yml | 39 +++++++++++++++++++ .github/workflows/v8upgrade.yml | 4 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/v8build_not_for_windows.yml diff --git a/.github/workflows/v8build_not_for_windows.yml b/.github/workflows/v8build_not_for_windows.yml new file mode 100644 index 000000000..74e03f659 --- /dev/null +++ b/.github/workflows/v8build_not_for_windows.yml @@ -0,0 +1,39 @@ +name: Automated V8 Build For MacOSX and Linux + +on: + pull_request: + branches: + - v8_*_upgrade + +jobs: + build: + name: Build V8 for ${{ matrix.platform }} + strategy: + matrix: + platform: [ubuntu-18.04, macos-latest] + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - name: Update depot_tools fetch config + run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + shell: bash + - name: Build V8 linux + if: matrix.platform == 'ubuntu-18.04' + run: cd deps && ./build.py --no-clang + - name: Build V8 macOS + if: matrix.platform == 'macos-latest' + run: cd deps && ./build.py + - name: Create PR + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update V8 static library for ${{ matrix.platform }} + branch: v8-lib + branch-suffix: random + delete-branch: true + title: V8 static library for ${{ matrix.platform }} + body: Auto-generated pull request to build V8 for ${{ matrix.platform }} + diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index b1de2596f..8fdbeb47e 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -1,6 +1,8 @@ name: V8 Upgrade -on: workflow_dispatch +on: + schedule: + - cron: '0 0 1 * *' # Run every 1st day of the month jobs: upgrade: From 63d18ad56af53ac83227de42cf927fa3874f2819 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Mon, 27 Sep 2021 16:27:08 +0200 Subject: [PATCH 03/11] Remove custom worflow for building macos and linux builds and use Github actions strategy.fail-fast attribute --- .github/workflows/v8build.yml | 8 +++- .github/workflows/v8build_not_for_windows.yml | 39 ------------------- 2 files changed, 6 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/v8build_not_for_windows.yml diff --git a/.github/workflows/v8build.yml b/.github/workflows/v8build.yml index 598bd48bb..14aec0fb4 100644 --- a/.github/workflows/v8build.yml +++ b/.github/workflows/v8build.yml @@ -1,11 +1,15 @@ name: V8 Build -on: workflow_dispatch - +on: + pull_request: + branches: + - v8_*_upgrade + jobs: build: name: Build V8 for ${{ matrix.platform }} strategy: + fail-fast: false matrix: platform: [ubuntu-18.04, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} diff --git a/.github/workflows/v8build_not_for_windows.yml b/.github/workflows/v8build_not_for_windows.yml deleted file mode 100644 index 74e03f659..000000000 --- a/.github/workflows/v8build_not_for_windows.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Automated V8 Build For MacOSX and Linux - -on: - pull_request: - branches: - - v8_*_upgrade - -jobs: - build: - name: Build V8 for ${{ matrix.platform }} - strategy: - matrix: - platform: [ubuntu-18.04, macos-latest] - runs-on: ${{ matrix.platform }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: true - fetch-depth: 1 - - name: Update depot_tools fetch config - run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* - shell: bash - - name: Build V8 linux - if: matrix.platform == 'ubuntu-18.04' - run: cd deps && ./build.py --no-clang - - name: Build V8 macOS - if: matrix.platform == 'macos-latest' - run: cd deps && ./build.py - - name: Create PR - uses: peter-evans/create-pull-request@v3 - with: - commit-message: Update V8 static library for ${{ matrix.platform }} - branch: v8-lib - branch-suffix: random - delete-branch: true - title: V8 static library for ${{ matrix.platform }} - body: Auto-generated pull request to build V8 for ${{ matrix.platform }} - From 945bd172813b996593c11a295814df774f05eab8 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Tue, 28 Sep 2021 13:12:01 +0200 Subject: [PATCH 04/11] Use custom pythin script to update the deps/include folder during the upgrade workflow --- .github/workflows/v8upgrade.yml | 4 +- deps/update_include_folder.py | 95 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100755 deps/update_include_folder.py diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index 8fdbeb47e..fd375e8a8 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -30,8 +30,8 @@ jobs: run: echo ::set-output name=VALUE::$(curl -sS 'https://omahaproxy.appspot.com/all.json?os=linux&channel=stable' | jq '.[0]["versions"][0]["v8_version"]') - name: Checkout v8 to stable commit ref run: cd deps/v8 && git fetch && git checkout ${{steps.v8_latest_commit.outputs.VALUE}} - - name: Copy deps/v8/include into deps/include - run: cp -r deps/v8/include/* deps/include/ + - name: Update include folder and cgo.go + run: cd deps && ./update_include_folder.py - name: Create PR metadata id: pr_metadata run: | diff --git a/deps/update_include_folder.py b/deps/update_include_folder.py new file mode 100755 index 000000000..1a5d3ac89 --- /dev/null +++ b/deps/update_include_folder.py @@ -0,0 +1,95 @@ +#!/usr/bin/python3 + +import os +import fnmatch +import pathlib +import shutil + +vendor_file_template = """// Package %s is required to provide support for vendoring modules +// DO NOT REMOVE +package %s +""" + +cgo_file_template = """// Copyright 2019 Roger Chapman and the v8go contributors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package v8go + +//go:generate clang-format -i --verbose -style=Chromium v8go.h v8go.cc + +// #cgo CXXFLAGS: -fno-rtti -fpic -std=c++14 -DV8_COMPRESS_POINTERS -DV8_31BIT_SMIS_ON_64BIT_ARCH -I${SRCDIR}/deps/include +// #cgo LDFLAGS: -pthread -lv8 +// #cgo darwin LDFLAGS: -L${SRCDIR}/deps/darwin_x86_64 +// #cgo linux LDFLAGS: -L${SRCDIR}/deps/linux_x86_64 +// #cgo windows LDFLAGS: -L${SRCDIR}/deps/windows_x86_64 -static -ldbghelp -lssp -lwinmm -lz +import "C" + +// These imports forces `go mod vendor` to pull in all the folders that +// contain V8 libraries and headers which otherwise would be ignored. +// DO NOT REMOVE +import ( + _ "rogchap.com/v8go/deps/darwin_x86_64" + _ "rogchap.com/v8go/deps/linux_x86_64" + _ "rogchap.com/v8go/deps/windows_x86_64" + _ "rogchap.com/v8go/deps/include" + %s +) +""" + +deps_path = os.path.dirname(os.path.realpath(__file__)) +v8go_path = os.path.abspath(os.path.join(deps_path, os.pardir)) +v8_include_path = os.path.join(deps_path, "v8", "include") +deps_include_path = os.path.join(deps_path, "include") + +def get_directories_names(path): + flist = [] + for p in pathlib.Path(path).iterdir(): + if p.is_dir(): + flist.append(p.name) + return flist + +def delete_files_except_vendor_files(path): + for rootDir, subdirs, filenames in os.walk(path): + # Find the files that matches the given patterm + for file in filenames: + if not fnmatch.fnmatch(file, '*.go'): + try: + os.remove(os.path.join(rootDir, file)) + except OSError: + print("Error while deleting file") + +def copy_include_directory(src_path, dest_path): + shutil.copytree(src_path, dest_path, dirs_exist_ok=True) + +def update_cgo_file(src_path): + directories = get_directories_names(src_path) + package_names = [] + + for directory_name in directories: + package_name = "_ \"rogchap.com/v8go/deps/include/" + directory_name + "\"\n" + package_names.append(package_name) + + with open(os.path.join(v8go_path, 'cgo.go'), 'w') as temp_file: + temp_file.write(cgo_file_template % (' '.join(package_names))) + +def create_new_directories_and_vendor_files(directories, src_path): + for directory in directories: + new_directory_path = os.path.join(src_path, directory) + os.mkdir(new_directory_path) + + with open(os.path.join(new_directory_path, 'vendor.go'), 'w') as temp_file: + temp_file.write(vendor_file_template % (directory, directory)) + + update_cgo_file(src_path) + + +v8_include_directories_names = get_directories_names(v8_include_path) +deps_include_directories_names = get_directories_names(deps_include_path) + +if not v8_include_directories_names == deps_include_directories_names: + new_directories = set(v8_include_directories_names).difference(set(deps_include_directories_names)) + create_new_directories_and_vendor_files(new_directories, deps_include_path) + +delete_files_except_vendor_files(deps_include_path) +copy_include_directory(v8_include_path, deps_include_path) From 355c61874e397a748e0c361c09e2aff69c08de8c Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Fri, 1 Oct 2021 12:59:42 +0200 Subject: [PATCH 05/11] Change workflow to run everyday and check the current v8 version with the latest v8 version --- .github/workflows/v8upgrade.yml | 6 +++++- deps/check_latest_v8_version.py | 31 +++++++++++++++++++++++++++++++ deps/v8_version | 1 + 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100755 deps/check_latest_v8_version.py create mode 100644 deps/v8_version diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index fd375e8a8..3290e8987 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -2,7 +2,7 @@ name: V8 Upgrade on: schedule: - - cron: '0 0 1 * *' # Run every 1st day of the month + - cron: '0 0 * * *' # Run every day jobs: upgrade: @@ -14,6 +14,8 @@ jobs: with: submodules: true fetch-depth: 0 + - name: Check latest v8 stable version + run: cd deps && ./check_latest_v8_version.py - name: Update depot_tools fetch config run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* shell: bash @@ -32,6 +34,8 @@ jobs: run: cd deps/v8 && git fetch && git checkout ${{steps.v8_latest_commit.outputs.VALUE}} - name: Update include folder and cgo.go run: cd deps && ./update_include_folder.py + - name: Update v8_version file + run: cd deps && rm v8_version && echo ${{steps.v8_latest_commit.outputs.VALUE}} >> v8_version - name: Create PR metadata id: pr_metadata run: | diff --git a/deps/check_latest_v8_version.py b/deps/check_latest_v8_version.py new file mode 100755 index 000000000..bf63725ef --- /dev/null +++ b/deps/check_latest_v8_version.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +# Script to compare current v8 version installed and the latest stable v8 version +# Returns 1 if the versions are the same +# Returns 0 if the latest stable version is higher than the current installed one + +import requests +import sys + +# api-endpoint +URL = "https://omahaproxy.appspot.com/all.json?os=linux&channel=stable" + +def compare_v8_version(current_version, new_version): + if current_version == new_version: + return 1 + else: + return 0 + +# Current version +v8_version_file = open("v8_version", "r") +current_v8_version_installed = v8_version_file.read().strip() + +# Get latest version +r = requests.get(url = URL) + +data = r.json() + +latest_stable_v8_version = data[0]["versions"][0]["v8_version"].strip() + +result = compare_v8_version(current_v8_version_installed, latest_stable_v8_version) +sys.exit(result) diff --git a/deps/v8_version b/deps/v8_version new file mode 100644 index 000000000..a98e9b2de --- /dev/null +++ b/deps/v8_version @@ -0,0 +1 @@ +9.0.257.18 From 7db1d65d09db6c740ccdb75b044bcd913c31db80 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Wed, 13 Oct 2021 22:47:22 -0400 Subject: [PATCH 06/11] move all the logic to the upgrade.py script --- .github/workflows/v8upgrade.yml | 20 +---- deps/check_latest_v8_version.py | 31 ------- deps/update_include_folder.py | 95 --------------------- deps/upgrade_v8.py | 144 ++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 143 deletions(-) delete mode 100755 deps/check_latest_v8_version.py delete mode 100755 deps/update_include_folder.py create mode 100755 deps/upgrade_v8.py diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index 3290e8987..852223415 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -14,32 +14,18 @@ jobs: with: submodules: true fetch-depth: 0 - - name: Check latest v8 stable version - run: cd deps && ./check_latest_v8_version.py - name: Update depot_tools fetch config run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* shell: bash - name: Add depot_tools to PATH run: echo "$PWD/deps/depot_tools" >> $GITHUB_PATH shell: bash - - name: Fetch V8 - run: cd deps && fetch v8 || true - - name: Get latest stable commit ref of v8 - id: v8_latest_commit - run: echo ::set-output name=VALUE::$(curl -sS 'https://omahaproxy.appspot.com/all.json?os=linux&channel=stable' | jq '.[0]["versions"][0]["v8_commit"]') - - name: Get latest stable version number of v8 - id: v8_latest_version - run: echo ::set-output name=VALUE::$(curl -sS 'https://omahaproxy.appspot.com/all.json?os=linux&channel=stable' | jq '.[0]["versions"][0]["v8_version"]') - - name: Checkout v8 to stable commit ref - run: cd deps/v8 && git fetch && git checkout ${{steps.v8_latest_commit.outputs.VALUE}} - - name: Update include folder and cgo.go - run: cd deps && ./update_include_folder.py - - name: Update v8_version file - run: cd deps && rm v8_version && echo ${{steps.v8_latest_commit.outputs.VALUE}} >> v8_version + - name: Run upgrade script + run: cd deps && ./upgrade_v8.py - name: Create PR metadata id: pr_metadata run: | - echo ::set-output name=pr_branch::"v8_${{steps.v8_latest_version.outputs.VALUE}}_upgrade" + echo ::set-output name=pr_branch::"v8_$(cat deps/v8_version)_upgrade" - name: Create PR uses: peter-evans/create-pull-request@v3 with: diff --git a/deps/check_latest_v8_version.py b/deps/check_latest_v8_version.py deleted file mode 100755 index bf63725ef..000000000 --- a/deps/check_latest_v8_version.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python3 - -# Script to compare current v8 version installed and the latest stable v8 version -# Returns 1 if the versions are the same -# Returns 0 if the latest stable version is higher than the current installed one - -import requests -import sys - -# api-endpoint -URL = "https://omahaproxy.appspot.com/all.json?os=linux&channel=stable" - -def compare_v8_version(current_version, new_version): - if current_version == new_version: - return 1 - else: - return 0 - -# Current version -v8_version_file = open("v8_version", "r") -current_v8_version_installed = v8_version_file.read().strip() - -# Get latest version -r = requests.get(url = URL) - -data = r.json() - -latest_stable_v8_version = data[0]["versions"][0]["v8_version"].strip() - -result = compare_v8_version(current_v8_version_installed, latest_stable_v8_version) -sys.exit(result) diff --git a/deps/update_include_folder.py b/deps/update_include_folder.py deleted file mode 100755 index 1a5d3ac89..000000000 --- a/deps/update_include_folder.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python3 - -import os -import fnmatch -import pathlib -import shutil - -vendor_file_template = """// Package %s is required to provide support for vendoring modules -// DO NOT REMOVE -package %s -""" - -cgo_file_template = """// Copyright 2019 Roger Chapman and the v8go contributors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package v8go - -//go:generate clang-format -i --verbose -style=Chromium v8go.h v8go.cc - -// #cgo CXXFLAGS: -fno-rtti -fpic -std=c++14 -DV8_COMPRESS_POINTERS -DV8_31BIT_SMIS_ON_64BIT_ARCH -I${SRCDIR}/deps/include -// #cgo LDFLAGS: -pthread -lv8 -// #cgo darwin LDFLAGS: -L${SRCDIR}/deps/darwin_x86_64 -// #cgo linux LDFLAGS: -L${SRCDIR}/deps/linux_x86_64 -// #cgo windows LDFLAGS: -L${SRCDIR}/deps/windows_x86_64 -static -ldbghelp -lssp -lwinmm -lz -import "C" - -// These imports forces `go mod vendor` to pull in all the folders that -// contain V8 libraries and headers which otherwise would be ignored. -// DO NOT REMOVE -import ( - _ "rogchap.com/v8go/deps/darwin_x86_64" - _ "rogchap.com/v8go/deps/linux_x86_64" - _ "rogchap.com/v8go/deps/windows_x86_64" - _ "rogchap.com/v8go/deps/include" - %s -) -""" - -deps_path = os.path.dirname(os.path.realpath(__file__)) -v8go_path = os.path.abspath(os.path.join(deps_path, os.pardir)) -v8_include_path = os.path.join(deps_path, "v8", "include") -deps_include_path = os.path.join(deps_path, "include") - -def get_directories_names(path): - flist = [] - for p in pathlib.Path(path).iterdir(): - if p.is_dir(): - flist.append(p.name) - return flist - -def delete_files_except_vendor_files(path): - for rootDir, subdirs, filenames in os.walk(path): - # Find the files that matches the given patterm - for file in filenames: - if not fnmatch.fnmatch(file, '*.go'): - try: - os.remove(os.path.join(rootDir, file)) - except OSError: - print("Error while deleting file") - -def copy_include_directory(src_path, dest_path): - shutil.copytree(src_path, dest_path, dirs_exist_ok=True) - -def update_cgo_file(src_path): - directories = get_directories_names(src_path) - package_names = [] - - for directory_name in directories: - package_name = "_ \"rogchap.com/v8go/deps/include/" + directory_name + "\"\n" - package_names.append(package_name) - - with open(os.path.join(v8go_path, 'cgo.go'), 'w') as temp_file: - temp_file.write(cgo_file_template % (' '.join(package_names))) - -def create_new_directories_and_vendor_files(directories, src_path): - for directory in directories: - new_directory_path = os.path.join(src_path, directory) - os.mkdir(new_directory_path) - - with open(os.path.join(new_directory_path, 'vendor.go'), 'w') as temp_file: - temp_file.write(vendor_file_template % (directory, directory)) - - update_cgo_file(src_path) - - -v8_include_directories_names = get_directories_names(v8_include_path) -deps_include_directories_names = get_directories_names(deps_include_path) - -if not v8_include_directories_names == deps_include_directories_names: - new_directories = set(v8_include_directories_names).difference(set(deps_include_directories_names)) - create_new_directories_and_vendor_files(new_directories, deps_include_path) - -delete_files_except_vendor_files(deps_include_path) -copy_include_directory(v8_include_path, deps_include_path) diff --git a/deps/upgrade_v8.py b/deps/upgrade_v8.py new file mode 100755 index 000000000..30308b63d --- /dev/null +++ b/deps/upgrade_v8.py @@ -0,0 +1,144 @@ +#!/usr/bin/python3 + +import urllib.request +import sys +import json +import subprocess +import os +import fnmatch +import pathlib +import shutil + +vendor_file_template = """// Package %s is required to provide support for vendoring modules +// DO NOT REMOVE +package %s +""" + +cgo_file_template = """// Copyright 2019 Roger Chapman and the v8go contributors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package v8go + +//go:generate clang-format -i --verbose -style=Chromium v8go.h v8go.cc + +// #cgo CXXFLAGS: -fno-rtti -fpic -std=c++14 -DV8_COMPRESS_POINTERS -DV8_31BIT_SMIS_ON_64BIT_ARCH -I${SRCDIR}/deps/include +// #cgo LDFLAGS: -pthread -lv8 +// #cgo darwin LDFLAGS: -L${SRCDIR}/deps/darwin_x86_64 +// #cgo linux LDFLAGS: -L${SRCDIR}/deps/linux_x86_64 +// #cgo windows LDFLAGS: -L${SRCDIR}/deps/windows_x86_64 -static -ldbghelp -lssp -lwinmm -lz +import "C" + +// These imports forces `go mod vendor` to pull in all the folders that +// contain V8 libraries and headers which otherwise would be ignored. +// DO NOT REMOVE +import ( + _ "rogchap.com/v8go/deps/darwin_x86_64" + _ "rogchap.com/v8go/deps/include" + %s + _ "rogchap.com/v8go/deps/linux_x86_64" + _ "rogchap.com/v8go/deps/windows_x86_64" +) +""" + +CHROME_VERSIONS_URL = "https://omahaproxy.appspot.com/all.json?os=linux&channel=stable" +V8_VERSION_FILE = "v8_version" + +deps_path = os.path.dirname(os.path.realpath(__file__)) +v8go_path = os.path.abspath(os.path.join(deps_path, os.pardir)) +env = os.environ.copy() +v8_path = os.path.join(deps_path, "v8", "include") +v8_include_path = os.path.join(deps_path, "v8", "include") +deps_include_path = os.path.join(deps_path, "include") + +def get_directories_names(path): + flist = [] + for p in pathlib.Path(path).iterdir(): + if p.is_dir(): + flist.append(p.name) + return sorted(flist) + +def delete_files_except_vendor_files(path): + for rootDir, subdirs, filenames in os.walk(path): + # Find the files that matches the given patterm + for file in filenames: + if not fnmatch.fnmatch(file, '*.go'): + try: + os.remove(os.path.join(rootDir, file)) + except OSError: + print("Error while deleting file") + +def package_name(package, index, total): + + name = f'_ "rogchap.com/v8go/deps/include/{package}"' + if (index + 1 == total): + return name + else: + return name + '\n' + +def update_cgo_file(src_path): + directories = get_directories_names(src_path) + package_names = [] + total_directories = len(directories) + + for index, directory_name in enumerate(directories): + package_names.append(package_name(directory_name, index, total_directories)) + + with open(os.path.join(v8go_path, 'cgo.go'), 'w') as temp_file: + temp_file.write(cgo_file_template % (' '.join(package_names))) + +def create_new_vendor_files(src_path): + directories = get_directories_names(src_path) + + for directory in directories: + directory_path = os.path.join(src_path, directory) + + vendor_go_file_path = os.path.join(directory_path, 'vendor.go') + + if os.path.isfile(vendor_go_file_path): + continue + + with open(os.path.join(directory_path, 'vendor.go'), 'w') as temp_file: + temp_file.write(vendor_file_template % (directory, directory)) + + update_cgo_file(src_path) + +def update_v8_file(src_path, version): + with open(os.path.join(src_path, V8_VERSION_FILE), "w") as v8_file: + v8_file.write(version) + +def read_v8_file(src_path): + v8_version_file = open(os.path.join(src_path, V8_VERSION_FILE), "r") + return v8_version_file.read().strip() + +def get_latest_v8_info(): + with urllib.request.urlopen(CHROME_VERSIONS_URL) as response: + json_response = response.read() + + return json.loads(json_response) + +# Current version +current_v8_version_installed = read_v8_file(deps_path) + +# Get latest version +latest_v8_info = get_latest_v8_info() + +latest_stable_v8_version = latest_v8_info[0]["versions"][0]["v8_version"] + +if not current_v8_version_installed == latest_stable_v8_version: + # fetch latest v8 + subprocess.run(["fetch", "v8"], + cwd=v8_path, + env=env) + subprocess.check_call(["git", "fetch"], + cwd=v8_path, + env=env) + # checkout latest stable commit + subprocess.check_call(["git", "checkout", latest_stable_v8_version], + cwd=v8_path, + env=env) + + delete_files_except_vendor_files(deps_include_path) + shutil.copytree(v8_include_path, deps_include_path, dirs_exist_ok=True) + create_new_vendor_files(deps_include_path) + update_v8_file(deps_path, latest_stable_v8_version) From a3dc326e589ebed9232cff3a162c2e0b85bbd1ae Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Thu, 14 Oct 2021 15:50:34 -0400 Subject: [PATCH 07/11] set python version in the GH action --- .github/workflows/v8upgrade.yml | 5 ++++- deps/upgrade_v8.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index 852223415..9281a3e8a 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -14,6 +14,9 @@ jobs: with: submodules: true fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' - name: Update depot_tools fetch config run: cd deps/depot_tools && git config --unset-all remote.origin.fetch; git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* shell: bash @@ -21,7 +24,7 @@ jobs: run: echo "$PWD/deps/depot_tools" >> $GITHUB_PATH shell: bash - name: Run upgrade script - run: cd deps && ./upgrade_v8.py + run: cd deps && python upgrade_v8.py - name: Create PR metadata id: pr_metadata run: | diff --git a/deps/upgrade_v8.py b/deps/upgrade_v8.py index 30308b63d..f7f6ce076 100755 --- a/deps/upgrade_v8.py +++ b/deps/upgrade_v8.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python import urllib.request import sys From 496ee27ed6dd1dea067a87c2cd2442f3dba9b035 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Fri, 15 Oct 2021 09:44:27 -0400 Subject: [PATCH 08/11] do not modify cgo.go file --- deps/upgrade_v8.py | 70 ++++++++++++---------------------------------- 1 file changed, 18 insertions(+), 52 deletions(-) diff --git a/deps/upgrade_v8.py b/deps/upgrade_v8.py index f7f6ce076..0e22bca1d 100755 --- a/deps/upgrade_v8.py +++ b/deps/upgrade_v8.py @@ -9,35 +9,17 @@ import pathlib import shutil -vendor_file_template = """// Package %s is required to provide support for vendoring modules -// DO NOT REMOVE +vendor_file_template = """// Generated by deps/upgrade_v8.py, DO NOT REMOVE/EDIT MANUALLY. +// Package %s is required to provide support for vendoring modules package %s """ -cgo_file_template = """// Copyright 2019 Roger Chapman and the v8go contributors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. +include_vendor_file_template = """// Generated by deps/upgrade_v8.py, DO NOT REMOVE/EDIT MANUALLY. +// Package include is required to provide support for vendoring modules +package include -package v8go - -//go:generate clang-format -i --verbose -style=Chromium v8go.h v8go.cc - -// #cgo CXXFLAGS: -fno-rtti -fpic -std=c++14 -DV8_COMPRESS_POINTERS -DV8_31BIT_SMIS_ON_64BIT_ARCH -I${SRCDIR}/deps/include -// #cgo LDFLAGS: -pthread -lv8 -// #cgo darwin LDFLAGS: -L${SRCDIR}/deps/darwin_x86_64 -// #cgo linux LDFLAGS: -L${SRCDIR}/deps/linux_x86_64 -// #cgo windows LDFLAGS: -L${SRCDIR}/deps/windows_x86_64 -static -ldbghelp -lssp -lwinmm -lz -import "C" - -// These imports forces `go mod vendor` to pull in all the folders that -// contain V8 libraries and headers which otherwise would be ignored. -// DO NOT REMOVE import ( - _ "rogchap.com/v8go/deps/darwin_x86_64" - _ "rogchap.com/v8go/deps/include" - %s - _ "rogchap.com/v8go/deps/linux_x86_64" - _ "rogchap.com/v8go/deps/windows_x86_64" + %s ) """ @@ -47,8 +29,8 @@ deps_path = os.path.dirname(os.path.realpath(__file__)) v8go_path = os.path.abspath(os.path.join(deps_path, os.pardir)) env = os.environ.copy() -v8_path = os.path.join(deps_path, "v8", "include") -v8_include_path = os.path.join(deps_path, "v8", "include") +v8_path = os.path.join(deps_path, "v8") +v8_include_path = os.path.join(v8_path, "include") deps_include_path = os.path.join(deps_path, "include") def get_directories_names(path): @@ -58,38 +40,28 @@ def get_directories_names(path): flist.append(p.name) return sorted(flist) -def delete_files_except_vendor_files(path): - for rootDir, subdirs, filenames in os.walk(path): - # Find the files that matches the given patterm - for file in filenames: - if not fnmatch.fnmatch(file, '*.go'): - try: - os.remove(os.path.join(rootDir, file)) - except OSError: - print("Error while deleting file") - def package_name(package, index, total): - name = f'_ "rogchap.com/v8go/deps/include/{package}"' if (index + 1 == total): return name else: return name + '\n' -def update_cgo_file(src_path): - directories = get_directories_names(src_path) +def create_include_vendor_file(src_path, directories): package_names = [] total_directories = len(directories) for index, directory_name in enumerate(directories): package_names.append(package_name(directory_name, index, total_directories)) - with open(os.path.join(v8go_path, 'cgo.go'), 'w') as temp_file: - temp_file.write(cgo_file_template % (' '.join(package_names))) + with open(os.path.join(src_path, 'vendor.go'), 'w') as temp_file: + temp_file.write(include_vendor_file_template % (' '.join(package_names))) -def create_new_vendor_files(src_path): +def create_vendor_files(src_path): directories = get_directories_names(src_path) + create_include_vendor_file(src_path, directories) + for directory in directories: directory_path = os.path.join(src_path, directory) @@ -101,8 +73,6 @@ def create_new_vendor_files(src_path): with open(os.path.join(directory_path, 'vendor.go'), 'w') as temp_file: temp_file.write(vendor_file_template % (directory, directory)) - update_cgo_file(src_path) - def update_v8_file(src_path, version): with open(os.path.join(src_path, V8_VERSION_FILE), "w") as v8_file: v8_file.write(version) @@ -125,12 +95,8 @@ def get_latest_v8_info(): latest_stable_v8_version = latest_v8_info[0]["versions"][0]["v8_version"] -if not current_v8_version_installed == latest_stable_v8_version: - # fetch latest v8 - subprocess.run(["fetch", "v8"], - cwd=v8_path, - env=env) - subprocess.check_call(["git", "fetch"], +if current_v8_version_installed != latest_stable_v8_version: + subprocess.check_call(["git", "fetch", "origin", latest_stable_v8_version], cwd=v8_path, env=env) # checkout latest stable commit @@ -138,7 +104,7 @@ def get_latest_v8_info(): cwd=v8_path, env=env) - delete_files_except_vendor_files(deps_include_path) + shutil.rmtree(deps_include_path) shutil.copytree(v8_include_path, deps_include_path, dirs_exist_ok=True) - create_new_vendor_files(deps_include_path) + create_vendor_files(deps_include_path) update_v8_file(deps_path, latest_stable_v8_version) From 5e0efdb98879ab369ac50e8fc936619a687c6b09 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Thu, 21 Oct 2021 16:16:11 -0400 Subject: [PATCH 09/11] Trigger upgrade and build workflow using dispatch --- .github/workflows/v8build.yml | 5 +---- .github/workflows/v8upgrade.yml | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/v8build.yml b/.github/workflows/v8build.yml index 14aec0fb4..d7efbfc7f 100644 --- a/.github/workflows/v8build.yml +++ b/.github/workflows/v8build.yml @@ -1,9 +1,6 @@ name: V8 Build -on: - pull_request: - branches: - - v8_*_upgrade +on: workflow_dispatch jobs: build: diff --git a/.github/workflows/v8upgrade.yml b/.github/workflows/v8upgrade.yml index 9281a3e8a..099a6a651 100644 --- a/.github/workflows/v8upgrade.yml +++ b/.github/workflows/v8upgrade.yml @@ -1,6 +1,7 @@ name: V8 Upgrade on: + workflow_dispatch: schedule: - cron: '0 0 * * *' # Run every day From b48997f03035895df4216e58d3c5cc38588b5a62 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Thu, 21 Oct 2021 16:41:22 -0400 Subject: [PATCH 10/11] Update README --- README.md | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 0030818b4..13bec15c9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Execute JavaScript from Go Github release -[![Go Report Card](https://goreportcard.com/badge/rogchap.com/v8go)](https://goreportcard.com/report/rogchap.com/v8go) +[![Go Report Card](https://goreportcard.com/badge/rogchap.com/v8go)](https://goreportcard.com/report/rogchap.com/v8go) [![Go Reference](https://pkg.go.dev/badge/rogchap.com/v8go.svg)](https://pkg.go.dev/rogchap.com/v8go) [![CI](https://github.com/rogchap/v8go/workflows/CI/badge.svg)](https://github.com/rogchap/v8go/actions?query=workflow%3ACI) ![V8 Build](https://github.com/rogchap/v8go/workflows/V8%20Build/badge.svg) @@ -106,7 +106,7 @@ case err := <- errs: // javascript error case <- time.After(200 * time.Milliseconds): vm := ctx.Isolate() // get the Isolate from the context - vm.TerminateExecution() // terminate the execution + vm.TerminateExecution() // terminate the execution err := <- errs // will get a termination error back from the running script } ``` @@ -177,7 +177,7 @@ To set this up: 2. Add the Mingw-w64 bin to your PATH environment variable (`C:\msys64\mingw64\bin` by default) 3. Open MSYS2 MSYS and execute `pacman -S mingw-w64-x86_64-toolchain` -V8 requires 64-bit on Windows, therefore it will not work on 32-bit systems. +V8 requires 64-bit on Windows, therefore it will not work on 32-bit systems. ## V8 dependency @@ -215,26 +215,16 @@ This project also aims to keep up-to-date with the latest (stable) release of V8 ### Upgrading the V8 binaries -This process is non-trivial, and hopefully we can automate more of this in the future. +We have the [upgradev8](https://github.com/rogchap/v8go/.github/workflow/v8upgrade.yml) workflow. +The workflow is triggered every day or manually. + +If the current [v8_version](https://github.com/rogchap/v8go/deps/v8_version) is different from the latest stable version, the workflow takes care of fetching the latest stable v8 files and copying them into `deps/include`. The last step of the workflow opens a new PR with the branch name `v8_upgrade/` with all the changes. + +The next steps are: -1) Make sure to clone the projects submodules (ie. the V8's `depot_tools` project): `git submodule update --init --recursive` -1) Add the `depot_tools` folder to your `PATH` eg: `export PATH=~/Development/rogchap/v8go/deps/depot_tools:$PATH` -1) From the `deps` folder run `fetch v8`; you only need to do this once, if you don't already have the V8 source. -1) Find the current stable release (`v8_version`) here: [https://omahaproxy.appspot.com](https://omahaproxy.appspot.com) -1) Create a new git branch from `master` eg. `git checkout -b v8_7_upgrade` -1) Enter the v8 folder and fetch all the latest git branches: `cd deps/v8 && git fetch` -1) Find the right `branch-heads/**` to checkout, for example if the `v8_version` is 8.7.220.31 then you want to `git checkout -branch-heads/8.7`. You can check all the `branch-heads` with `git branch --remotes | grep branch-heads/` -1) Copy all the contents of `deps/v8/include` to `deps/include` making sure not to delete any of the `vendor.go` files, -which are required for users that are using `go mod vendor`. If there are any new folders added, make sure to create new -`vendor.go` files in each folder within `deps/include` and update `cgo.go`. -1) Optionally build the V8 binary for your OS: `cd deps && ./build.py`. V8 is a large project, and building the binary -can take up to 30 minutes. Once built all the tests should still pass via `go test -v .` -1) Commit your changes, making sure that the git submodules have been updated to the new checksum for the version of V8. -Make sure *NOT* to add your build of the binary, as this will be build via CI. 1) Because the build is so long, this is not automatically triggered. Go to the [V8 Build](https://github.com/rogchap/v8go/actions?query=workflow%3A%22V8+Build%22) Github Action, Select "Run workflow", -and select your pushed branch eg. `v8_7_upgrade`. +and select your pushed branch eg. `v8_upgrade/`. 1) Once built, this should open 3 PRs against your branch to add the `libv8.a` for Linux, macOS and Windows; merge these PRs into your branch. You are now ready to raise the PR against `master` with the latest version of V8. From 256a7c784f8d43f8d85f1f3ea9e1046f8555e7ad Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Fri, 22 Oct 2021 12:10:00 -0400 Subject: [PATCH 11/11] Apply feedback on README and script function names --- README.md | 2 +- deps/upgrade_v8.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 13bec15c9..7fe557d56 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ If the current [v8_version](https://github.com/rogchap/v8go/deps/v8_version) is The next steps are: -1) Because the build is so long, this is not automatically triggered. Go to the [V8 +1) The build is not yet triggered automatically. To trigger it manually, go to the [V8 Build](https://github.com/rogchap/v8go/actions?query=workflow%3A%22V8+Build%22) Github Action, Select "Run workflow", and select your pushed branch eg. `v8_upgrade/`. 1) Once built, this should open 3 PRs against your branch to add the `libv8.a` for Linux, macOS and Windows; merge diff --git a/deps/upgrade_v8.py b/deps/upgrade_v8.py index 0e22bca1d..41d712e02 100755 --- a/deps/upgrade_v8.py +++ b/deps/upgrade_v8.py @@ -73,11 +73,11 @@ def create_vendor_files(src_path): with open(os.path.join(directory_path, 'vendor.go'), 'w') as temp_file: temp_file.write(vendor_file_template % (directory, directory)) -def update_v8_file(src_path, version): +def update_v8_version_file(src_path, version): with open(os.path.join(src_path, V8_VERSION_FILE), "w") as v8_file: v8_file.write(version) -def read_v8_file(src_path): +def read_v8_version_file(src_path): v8_version_file = open(os.path.join(src_path, V8_VERSION_FILE), "r") return v8_version_file.read().strip() @@ -88,7 +88,7 @@ def get_latest_v8_info(): return json.loads(json_response) # Current version -current_v8_version_installed = read_v8_file(deps_path) +current_v8_version_installed = read_v8_version_file(deps_path) # Get latest version latest_v8_info = get_latest_v8_info() @@ -107,4 +107,4 @@ def get_latest_v8_info(): shutil.rmtree(deps_include_path) shutil.copytree(v8_include_path, deps_include_path, dirs_exist_ok=True) create_vendor_files(deps_include_path) - update_v8_file(deps_path, latest_stable_v8_version) + update_v8_version_file(deps_path, latest_stable_v8_version)