diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 000000000..f184f49d2 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,89 @@ +# Versioning + +SecretFlow artifacts belongs to one of the following categories: + +- "Releases", i.e. artifact versions follow the **x.y.z** format, where **x** is called "major version", **y** is called "minor version", **z** is called "patch version". We follow [Semantic Versioning 2.0.0](https://semver.org/) to update versions. e.g. +``` +1.2.3 +``` + +- "Pre-releases", i.e. artifact versions follow the **x.y.zb0** format. **Pre-release segment** is fixed to **b0**. e.g. +``` +0.8.7b0 # Pre-release of 0.8.7 +``` + +- "Developmental releases", i.e. artifact versions follow the **x.y.z.devYYMMDD** or **x.y.z.devYYMMDD** format, where **YY** is the last two digits of year(00-99), **MM** is the zero-padded digits of month(01-12) and **DD** is zero-padded digits of day of the month(01-31). **YY**, **MM** and **DD** indicates the date when a Developmental release is published. e.g. +``` +1.1.0.dev230820 # Developmental release of 1.1.0 published at 23/08/20 +1.3.0.dev231115 # Developmental release of 1.3.0 published at 23/11/15 +``` + +- "Post-releases", i.e. artifact versions follow the **x.y.zb0.postN** or **x.y.z.postN** format. **x.y.zb0.postN** is a Developmental release of a Pre-release while **x.y.z.postN** is a Developmental release of a Release. e.g. +``` +1.1.0.post0 # The 1st Post-release of 1.1.0 +1.3.0b0.post9 # The 10th Post-release of 1.3.0b0 +``` + +## Releases + +Release **x.y.z** meets the following conditions: +- Pre-release **x.y.zb0** has been published before, and +- Pre-release **x.y.zb0** has passed through evaluation. + +## Pre-releases + +Pre-release **x.y.zb0** is to support testing by external users prior to release **x.y.z**. + +## Developmental releases + +Developmental releases are to provide the preview of new features. They are considered unstable. + +## Post-releases + +Post-releases are to address minor errors of Releases and Pre-releases. They are considered unstable. + +# 版本控制 + +SecretFlow 的 artifact 属于以下几种类别之一: + +- "Releases",即 artifact 版本遵循 **x.y.z** 格式,其中 **x** 称为 "主版本", **y** 称为 "次版本", **z** 称为 "修复版本"。我们遵循 [Semantic Versioning 2.0.0](https://semver.org/) 更新版本。例如: +``` +1.2.3 +``` + +- "Pre-releases",即 artifact 版本遵循 **x.y.zb0** 格式。预发布字段固定为 **b0**。例如: +``` +0.8.7b0 # 0.8.7的Pre-release +``` + +- "Developmental releases",即 artifact 版本遵循 **x.y.z.devYYMMDD** 或 **x.y.z.devYYMMDD** 格式,其中 **YY** 是年份的最后两位(00-99), **MM** 是月份的零填充数字(01-12), **DD** 是当月日期的零填充数字(01-31)。**YY**, **MM** 和 **DD** 用以表示发布开发版的日期。例如: +``` +1.1.0.dev230820 # 于23/08/20发布的1.1.0开发版 +1.3.0.dev231115 # 于23/11/15发布的1.3.0开发版 +``` + + +- "Post-releases",即 artifact 版本遵循 **x.y.zb0.postN** 或 **x.y.z.postN** 格式。 **x.y.zb0.postN** 是 Pre-releases 的 Post-releases,而 x.y.z.postN 是 Releases 的Post-releases 。例如: +``` +1.1.0.post0 # 1.1.0的第1个Post-release +1.3.0b0.post9 # 1.3.0b0的第10个Post-release +``` + +## Releases + +Release **x.y.z** 满足以下条件: + +- Pre-release **x.y.zb0** 已经发布,并且 +- Pre-release **x.y.zb0** 已经经过测试评估。 + +## Pre-releases + +Pre-release **x.y.zb0** 是对外提供的 Release **x.y.z** 的测试版本。 + +## Developmental releases + +Developmental releases 旨在提供新功能的预览。它们被认为是不稳定的。 + +## Post-releases + +Post-releases 用于解决 Releases 和 Pre-releases 的轻微错误。它们被认为是不稳定的。 diff --git a/docker/dev/.gitignore b/docker/dev/.gitignore new file mode 100644 index 000000000..6e71b056c --- /dev/null +++ b/docker/dev/.gitignore @@ -0,0 +1,3 @@ +*.whl +*.yml +*.json diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile new file mode 100644 index 000000000..6431f3b91 --- /dev/null +++ b/docker/dev/Dockerfile @@ -0,0 +1,48 @@ +FROM secretflow/base-ci:latest as builder + +RUN yum install -y \ + wget autoconf bison flex git protobuf-devel libnl3-devel \ + libtool make pkg-config protobuf-compiler \ + && yum clean all + +RUN cd / && git clone https://github.com/google/nsjail.git \ + && cd /nsjail && git checkout 3.3 -b v3.3 \ + && make && mv /nsjail/nsjail /bin + +FROM secretflow/anolis8-python:3.8.15 as python + +FROM openanolis/anolisos:8.8 + +LABEL maintainer="secretflow-contact@service.alipay.com" + +COPY --from=builder /bin/nsjail /usr/local/bin/ +COPY --from=python /root/miniconda3/envs/secretflow/bin/ /usr/local/bin/ +COPY --from=python /root/miniconda3/envs/secretflow/lib/ /usr/local/lib/ + +RUN yum install -y protobuf libnl3 && yum clean all + +RUN grep -rl '#!/root/miniconda3/envs/secretflow/bin' /usr/local/bin/ | xargs sed -i -e 's/#!\/root\/miniconda3\/envs\/secretflow/#!\/usr\/local/g' + +COPY *.whl /tmp/ + +RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ kuscia +RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ /tmp/*.whl && rm -rf /root/.cache + +COPY .nsjail /root/.nsjail + +ARG config_templates="" +LABEL kuscia.secretflow.config-templates=$config_templates + +ARG deploy_templates="" +LABEL kuscia.secretflow.deploy-templates=$deploy_templates + +ARG comp_list="" +LABEL kuscia.secretflow.comp_list=$comp_list + +ARG translation="" +LABEL kuscia.secretflow.translation=$translation + +# run as root for now +WORKDIR /root + +CMD ["/bin/bash"] diff --git a/docker/dev/build.sh b/docker/dev/build.sh new file mode 100644 index 000000000..a79b01f1c --- /dev/null +++ b/docker/dev/build.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -e + +show_help() { + echo "Usage: bash build.sh [OPTION]... -v {the_version}" + echo " -v --version" + echo " the version to build with." + echo " -l --latest" + echo " tag this version as latest." + echo " -u --upload" + echo " upload to docker registry." +} + +if [[ "$#" -lt 2 ]]; then + show_help + exit +fi + +while [[ "$#" -ge 1 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift + if [[ "$#" -eq 0 ]]; then + echo "Version shall not be empty." + echo "" + show_help + exit 1 + fi + shift + ;; + -l|--latest) + LATEST=1 + shift + ;; + -u|--upload) + UPLOAD=1 + shift + ;; + *) + echo "Unknown argument passed: $1" + exit 1 + ;; + esac +done + + +if [[ -z ${VERSION} ]]; then + echo "Please specify the version." + exit 1 +fi + +GREEN="\033[32m" +NO_COLOR="\033[0m" + +DOCKER_REG="secretflow" + + +IMAGE_TAG=${DOCKER_REG}/sf-dev-anolis8:${VERSION} +LATEST_TAG=${DOCKER_REG}/sf-dev-anolis8:latest +echo -e "Building ${GREEN}${IMAGE_TAG}${NO_COLOR}" +(cd ../.. && rm -rf dist/ && python setup.py bdist_wheel --lite && cp -r dist/* docker/dev/) +(cd ../ && cp -r release/.nsjail dev/ && cp release/.condarc dev/ && cp *.yml dev/ && cp *.json dev/) +docker build . -f Dockerfile -t ${IMAGE_TAG} --build-arg config_templates="$(cat config_templates.yml)" --build-arg deploy_templates="$(cat deploy_templates.yml)" --build-arg comp_list="$(cat comp_list.json)" --build-arg translation="$(cat translation.json)" +echo -e "Finish building ${GREEN}${IMAGE_TAG}${NO_COLOR}" +rm -rf .nsjail +rm -f .condarc +rm -f *.whl +if [[ UPLOAD -eq 1 ]]; then + docker push ${IMAGE_TAG} +fi + + +if [[ LATEST -eq 1 ]]; then + echo -e "Tag and push ${GREEN}${LATEST_TAG}${NO_COLOR} ..." + docker tag ${IMAGE_TAG} ${LATEST_TAG} + if [[ UPLOAD -eq 1 ]]; then + docker push ${LATEST_TAG} + fi +fi diff --git a/secretflow/component/data_utils.py b/secretflow/component/data_utils.py index bd10d55ee..2d3ffd6b2 100644 --- a/secretflow/component/data_utils.py +++ b/secretflow/component/data_utils.py @@ -215,7 +215,7 @@ def extract_table_header( if load_ids: for t, h in zip(slice.id_types, slice.ids): if col_selects is not None: - if id not in col_selects: + if h not in col_selects: # id not selected, skip continue col_selects.remove(h)