cmdline: support protoc-gen-validate (#12)

* cmdline: support protoc-gen-validate * update readme * fix test * fix test * fix typos * add readme
trpc-group · Oct 20, 2023 · 7f44a06 · 7f44a06
1 parent 37d7d5c
commit 7f44a06
Show file tree

Hide file tree

Showing 24 changed files with 1,114 additions and 29 deletions.
diff --git a/README.md b/README.md
@@ -44,13 +44,13 @@ $ export PATH=~/go/bin:$PATH # Add this to your `~/.bashrc`.
 
 ### Dependencies
 
- <!-- by using one of the following methods.
+Use one of the following methods to download:
 
 #### Using trpc setup
 
 After installation of trpc-cmdline, simply running `trpc setup` will automatically install all the dependencies. 
 
-#### Install separately -->
+#### Install separately
 
 <details><summary>Install protoc </summary><br><pre>
 $ # Reference: https://grpc.io/docs/protoc-installation/
@@ -85,6 +85,9 @@ $ # Reference: https://github.com/uber-go/mock
 $ go install go.uber.org/mock/mockgen@latest
 </pre></details>
 
+<details><summary>Install protoc-gen-validate</summary><br><pre>
+$ Please download the binaries in https://github.com/bufbuild/protoc-gen-validate/releases
+</pre></details>
 
 ## Quick Start
 
@@ -190,6 +193,7 @@ The following lists some frequently used flags.
 * `-d some-dir`: Search paths for pb files (including dependent pb files), can be specified multiple times.
 * `--mock=false`: Disable generation of mock stub code.
 * `--nogomod=true`: Do not generate go.mod file in the stub code, only effective when --rpconly=true, defaults to false.
+* `--validate=true`: Enables data validation. For detailed usage, see [/docs/examples/example-2/README.md](/docs/examples/example-2/README.md)
 
 For additional flags please run `trpc -h` and `trpc [subcmd] -h`.
 

diff --git a/README.zh_CN.md b/README.zh_CN.md
@@ -44,13 +44,13 @@ $ export PATH=~/go/bin:$PATH # Add this to your `~/.bashrc`.
 
 ### 安装依赖
 
- <!-- by using one of the following methods.
+ by using one of the following methods.
 
-#### Using trpc setup
+#### 使用 trpc setup 一键安装所有依赖
 
-After installation of trpc-cmdline, simply running `trpc setup` will automatically install all the dependencies. 
+只需要运行 `trpc setup` 便可安装所有依赖。假如有些依赖安装不成功，可以参考下一节进行手动安装。
 
-#### Install separately -->
+#### 手动安装各种依赖
 
 <details><summary>Install protoc </summary><br><pre>
 $ # Reference: https://grpc.io/docs/protoc-installation/
@@ -85,6 +85,9 @@ $ # Reference: https://github.com/uber-go/mock
 $ go install go.uber.org/mock/mockgen@latest
 </pre></details>
 
+<details><summary>Install protoc-gen-validate</summary><br><pre>
+$ Please download the binaries in https://github.com/bufbuild/protoc-gen-validate/releases
+</pre></details>
 
 ## 快速上手
 
@@ -190,6 +193,7 @@ out
 * `-d some-dir`: 添加 proto 文件的查找路径（包括依赖的 proto 文件），可以指定多次
 * `--mock=false`: 禁止生成 mock 代码
 * `--nogomod=true`: 在生成桩代码时不生成 `go.mod` 文件，只在 `--rpconly=true` 的时候生效, 默认为 `false`
+* `--validate=true`: 开启数据校验，详细用法见 [/docs/examples/example-2/README.zh_CN.md](/docs/examples/example-2/README.zh_CN.md)
 
 更多命令行选项可以执行 `trpc -h` 以及 `trpc [subcmd] -h` 来进行查看。
 

diff --git a/cmd/create/cmdflags.go b/cmd/create/cmdflags.go
@@ -39,8 +39,10 @@ func AddCreateFlags(createCmd *cobra.Command) {
 		"Whether to generate stub code for dependencies, only effective when --rpconly=true, defaults to false")
 	createCmd.Flags().Bool("nogomod", false,
 		"Do not generate go.mod file in the stub code, only effective when --rpconly=true, defaults to false")
-	createCmd.Flags().Bool("secvenabled", true,
-		"Enable generation of validate.go file using protoc-gen-secv, defaults to true")
+	createCmd.Flags().Bool("secvenabled", false,
+		"Enable generation of validate.go file using protoc-gen-secv, defaults to false")
+	createCmd.Flags().Bool("validate", false,
+		"Enable generation of validate.pb.go file using protoc-gen-validate, defaults to false")
 	createCmd.Flags().String("kvfile", "",
 		"Provide a json file path to unmarshal into key-value pairs (KVs) for usage in template files")
 	createCmd.Flags().String("kvrawjson", "",

diff --git a/cmd/create/create_test.go b/cmd/create/create_test.go
@@ -54,6 +54,7 @@ type testcase struct {
 	alias         bool
 	lang          string
 	wantErr       bool
+	opts          []string
 }
 
 func Test_CreateCmd(t *testing.T) {
@@ -130,6 +131,11 @@ func Test_CreateCmd(t *testing.T) {
 			pbdir:   "9-restful",
 			pbfile:  "helloworld.proto",
 			rpconly: true,
+		}, {
+			name:   "10-validate-pgv",
+			pbdir:  "10-validate-pgv",
+			pbfile: "helloworld.proto",
+			opts:   []string{"--validate"},
 		},
 	}
 
@@ -161,7 +167,7 @@ func Test_CreateCmd(t *testing.T) {
 				panic("walk testcase error")
 			}
 
-			opts := []string{}
+			opts := tt.opts
 			for _, d := range dirs {
 				opts = append(opts, "--protodir", d)
 			}

diff --git a/cmd/create/generate.go b/cmd/create/generate.go
@@ -141,6 +141,15 @@ func runProtoc(fd *FD, pbOutDir string, option *params.Option) ([]string, error)
 			return nil, fmt.Errorf("run protoc-gen-secv for %s err: %w", option.Protofile, err)
 		}
 	}
+	// Run protoc-gen-validate.
+	if support, ok := config.SupportValidate[option.Language]; ok && support && option.ValidateEnabled {
+		if err := pb.Protoc(
+			option.Protodirs, option.Protofile, option.Language, pbOutDir,
+			append(opts, pb.WithValidateEnabled(true))...,
+		); err != nil {
+			return nil, fmt.Errorf("run protoc-gen-validate for %s err: %w", option.Protofile, err)
+		}
+	}
 	log.Debug("pbOutDir = %s", pbOutDir)
 	// collect the generated files
 	matches, err := filepath.Glob(pbOutDir)
@@ -348,6 +357,15 @@ func (g *genDependencyRPCStubParam) genDependencyRPCStubPB() error {
 		if err := pb.Protoc(
 			searchPath, g.fname, g.option.Language, g.outputDir,
 			append(opts, pb.WithSecvEnabled(true))...,
+		); err != nil {
+			return fmt.Errorf("generate secv file for %s err: %w", g.fname, err)
+		}
+	}
+	// Run protoc-gen-validate.
+	if support, ok := config.SupportValidate[g.option.Language]; ok && support && g.option.ValidateEnabled {
+		if err := pb.Protoc(
+			searchPath, g.fname, g.option.Language, g.outputDir,
+			append(opts, pb.WithValidateEnabled(true))...,
 		); err != nil {
 			return fmt.Errorf("generate validation file for %s err: %w", g.fname, err)
 		}

diff --git a/cmd/create/options.go b/cmd/create/options.go
@@ -323,6 +323,10 @@ func (c *Create) parseOutputOptions(flags *pflag.FlagSet) error {
 	if err != nil {
 		return fmt.Errorf("flags parse secvenabled bool err: %w", err)
 	}
+	c.options.ValidateEnabled, err = flags.GetBool("validate")
+	if err != nil {
+		return fmt.Errorf("flags parse validate bool err: %w", err)
+	}
 	kvFile, err := flags.GetString("kvfile")
 	if err != nil {
 		return fmt.Errorf("flags parse kvfile string err: %w", err)

diff --git a/docs/README.md b/docs/README.md
@@ -49,4 +49,5 @@ The same applies to other languages such as C++, you need to specify `--lang=cpp
 * [example2](examples/example-2/README.md) shows how to use pb option extension features, such as
   * How to add aliases for service names
   * How to add custom tags for fields
+  * How to generate validate.pb.go file
   * How to generate swagger/openapi documentation
diff --git a/docs/README.zh_CN.md b/docs/README.zh_CN.md
@@ -49,5 +49,6 @@ trpc create -p hello.proto --assetdir=~/.trpc-cmdline-assets/protobuf/asset_go
 * [example2](examples/example-2/README.zh_CN.md) 展示了如何使用 pb option 扩展功能，例如
   * 如何为服务名添加别名
   * 如何为字段添加自定义 tag
+  * 如何生成 validate.pb.go 文件
   * 如何生成 swagger/openapi 文档
   * ...
diff --git a/docs/examples/example-2/README.md b/docs/examples/example-2/README.md
@@ -58,6 +58,49 @@ And when executing `trpc create`, you need to specify the `--gotag` option:
 trpc create -p helloworld.proto -o out --gotag
 ```
 
+## Generating validate.pb.go File
+
+For a complete example, see [/testcase/create/10-validate-pgv/helloworld.proto](/testcase/create/10-validate-pgv/helloworld.proto).
+
+This feature requires the installation of [protoc-gen-validate](https://github.com/bufbuild/protoc-gen-validate). Typically, `trpc setup` can automatically install these dependencies.
+
+Here is an example of a proto file:
+
+```proto
+syntax = "proto3";
+package helloworld;
+
+option go_package="trpc.group/some-example/helloworld";
+
+import "validate/validate.proto";
+
+service HelloSvr {
+    rpc Hello(HelloRequest) returns(HelloResponse);
+}
+
+message HelloRequest {
+    string msg = 1 [(validate.rules).string.email=true];
+}
+
+message HelloResponse {
+    int32 err_code = 1; 
+    string err_msg = 2; 
+}
+```
+
+Points to note:
+
+* The reference is `import "validate/validate.proto";`. You can download this file from the [protoc-gen-validate repository](https://github.com/bufbuild/protoc-gen-validate/blob/main/validate/validate.proto) and specify the path (download this file to `somedir/validate/validate.proto` and specify `-d somedir`). The trpc-cmdline tool has a built-in copy of this file.
+* The syntax for the validation rule `[(validate.rules).string.email=true]` can be found in the [protoc-gen-validate documentation](https://github.com/bufbuild/protoc-gen-validate/blob/v1.0.2/README.md).
+* When generating code, you need to add `--validate=true`, like this:
+  ```shell
+  trpc create -p helloworld.proto -o out --validate
+  ```
+* The generated stub code will include the `xxx.validate.pb.go` file.
+* In the generated project code, the following two locations will automatically add validate-related plugin information:
+  * All `main.go` will add an anonymous reference `import _ "trpc.group/trpc-filter/validation"`
+  * In `trpc_go.yaml`, the client/server filter configuration will include `- validation`
+
 ## Generating swagger/openapi Documentation
 
 The trpc-cmdline tool provides the `trpc apidocs` subcommand to generate documentation. Users can execute `trpc apidocs -h` to view all supported command options.

diff --git a/docs/examples/example-2/README.zh_CN.md b/docs/examples/example-2/README.zh_CN.md
@@ -58,6 +58,49 @@ message Req{
 trpc create -p helloworld.proto -o out --gotag
 ```
 
+## 生成 validate.pb.go 文件
+
+完整示例见 [/testcase/create/10-validate-pgv/helloworld.proto](/testcase/create/10-validate-pgv/helloworld.proto)。
+
+本功能需要安装 [protoc-gen-validate](https://github.com/bufbuild/protoc-gen-validate)，通常来说 `trpc setup` 即可使这些依赖被自动安装完毕。
+
+示例 proto 文件如下：
+
+```proto
+syntax = "proto3";
+package helloworld;
+
+option go_package="trpc.group/some-example/helloworld";
+
+import "validate/validate.proto";
+
+service HelloSvr {
+    rpc Hello(HelloRequest) returns(HelloResponse);
+}
+
+message HelloRequest {
+    string msg = 1 [(validate.rules).string.email=true];
+}
+
+message HelloResponse {
+    int32 err_code = 1; 
+    string err_msg = 2; 
+}
+```
+
+注意事项：
+
+* 引用为 `import "validate/validate.proto";`，这个文件可以自行从 [protoc-gen-validate 仓库](https://github.com/bufbuild/protoc-gen-validate/blob/main/validate/validate.proto) 进行下载并指定路径（将这个文件下载到 `somedir/validate/validate.proto` 然后指定 `-d somedir`），trpc-cmdline 工具内置了一份该文件。
+* 校验规则 ` [(validate.rules).string.email=true]` 的写法可以参考 [protoc-gen-validate 文档](https://github.com/bufbuild/protoc-gen-validate/blob/v1.0.2/README.md)。
+* 在生成代码时需要加上 `--validate=true`，如
+  ```shell
+  trpc create -p helloworld.proto -o out --validate
+  ```
+* 此时生成的桩代码会包含 `xxx.validate.pb.go` 文件
+* 在生成的项目代码中，有以下两个位置会自动添加 validate 相关的插件信息
+  * 所有的 `main.go` 中会添加匿名引用 `import _ "trpc.group/trpc-filter/validation"`
+  * 在 `trpc_go.yaml` 的 client/server 的 filter 配置项会存在 `- validation` 一项
+
 ## 生成 swagger/openapi 文档
 
 trpc-cmdline 工具提供了 `trpc apidocs` 子命令以生成文档，用户可以执行 `trpc apidocs -h` 以查看所有支持的命令选项。

diff --git a/gobin/assets.go b/gobin/assets.go
diff --git a/install/protobuf/asset_go/cmd/client/main.go.tpl b/install/protobuf/asset_go/cmd/client/main.go.tpl
@@ -16,6 +16,9 @@ import (
 	"{{$domainName}}/{{$groupName}}/trpc-go{{$versionSuffix}}/client"
 	"{{$domainName}}/{{$groupName}}/trpc-go{{$versionSuffix}}/log"
 	_ "{{ $domainName }}/{{ $groupName }}/trpc-filter/debuglog{{ $versionSuffix }}"
+	{{- if (or .ValidateEnabled .SecvEnabled)  }}
+	_ "{{ $domainName }}/{{ $groupName }}/trpc-filter/validation{{ $versionSuffix }}"
+	{{- end }}
 	pb "{{ trimright ";" $goPkgName }}"
 	{{ range $.ImportsX }}
 		{{.Name}} "{{.Path}}"

diff --git a/install/protobuf/asset_go/go.mod.tpl b/install/protobuf/asset_go/go.mod.tpl
@@ -35,6 +35,7 @@ replace {{$rpcdir}} => ./stub/{{$rpcdir}}
           (not (hasprefix "trpc.group/trpc/trpc-protocol" $v))
           (not (hasprefix "trpc.group/wineguo/trpc-protocol" $v))
           (not (hasprefix "github.com/golang/protobuf" $v)) 
+          (not (hasprefix "github.com/envoyproxy/protoc-gen-validate" $v)) 
           (ne $v "trpc.group/devsec/protoc-gen-secv/v2/validate")
           (ne $v "trpc.group/devsec/protoc-gen-secv/validate")
           (not (hasprefix "google/protobuf/" $k)) }}

diff --git a/install/protobuf/asset_go/main.go.tpl b/install/protobuf/asset_go/main.go.tpl
@@ -19,6 +19,9 @@ import (
 	{{- end }}
 	_ "{{$domainName}}/{{$groupName}}/trpc-filter/debuglog{{$versionSuffix}}"
 	_ "{{$domainName}}/{{$groupName}}/trpc-filter/recovery{{$versionSuffix}}"
+	{{- if (or .ValidateEnabled .SecvEnabled)  }}
+	_ "{{ $domainName }}/{{ $groupName }}/trpc-filter/validation{{ $versionSuffix }}"
+	{{- end }}
 	trpc "{{$domainName}}/{{$groupName}}/trpc-go{{$versionSuffix}}"
 	"{{$domainName}}/{{$groupName}}/trpc-go{{$versionSuffix}}/log"
     {{ if ne $goPkgOption "" -}}

diff --git a/install/protobuf/asset_go/trpc_go.yaml b/install/protobuf/asset_go/trpc_go.yaml
@@ -20,6 +20,9 @@ server:  # Server configuration.
   filter:  # List of interceptors for all service handler functions.
     - simpledebuglog
     - recovery  # Intercept panics from business processing goroutines created by the framework.
+    {{- if (or .ValidateEnabled .SecvEnabled)  }}
+    - validation
+    {{- end }}
   service:  # Services provided by the business, can have multiple.
     {{range $index, $service := .Services}}
     {{- $serviceName := $service.Name -}}
@@ -41,6 +44,9 @@ client:  # Backend configuration for client calls.
   namespace: Development  # Environment for all backends.
   filter:  # List of interceptors for all backend function calls.
     - simpledebuglog
+    {{- if (or .ValidateEnabled .SecvEnabled)  }}
+    - validation
+   {{- end }}
   service:  # Configuration for individual backends.
     {{range $index, $service := .Services}}
     {{- $serviceName := $service.Name -}}

diff --git a/install/submodules/trpc-protocol/pb/go/trpc/validate/validate.pb.go b/install/submodules/trpc-protocol/pb/go/trpc/validate/validate.pb.go
diff --git a/install/submodules/trpc-protocol/trpc/validate/validate.proto b/install/submodules/trpc-protocol/trpc/validate/validate.proto
@@ -535,7 +535,7 @@ message StringRules {
     // at a maximum
     optional uint64 max_bytes = 5;
 
-    // Pattern specifes that this field must match against the specified
+    // Pattern specifies that this field must match against the specified
     // regular expression (RE2 syntax). The included expression should elide
     // any delimiters.
     optional string pattern  = 6;
@@ -659,7 +659,7 @@ message BytesRules {
     // at a maximum
     optional uint64 max_len = 3;
 
-    // Pattern specifes that this field must match against the specified
+    // Pattern specifies that this field must match against the specified
     // regular expression (RE2 syntax). The included expression should elide
     // any delimiters.
     optional string pattern  = 4;
@@ -745,11 +745,11 @@ message RepeatedRules {
     optional uint64 max_items = 2;
 
     // Unique specifies that all elements in this field must be unique. This
-    // contraint is only applicable to scalar and enum types (messages are not
+    // constraint is only applicable to scalar and enum types (messages are not
     // supported).
     optional bool   unique    = 3;
 
-    // Items specifies the contraints to be applied to each item in the field.
+    // Items specifies the constraints to be applied to each item in the field.
     // Repeated message fields will still execute validation against each item
     // unless skip is specified here.
     optional FieldRules items = 4;

diff --git a/install/trpc.yaml b/install/trpc.yaml
@@ -29,6 +29,20 @@ tools:
       md5: ""
       repository: "github.com/golang/protobuf/protoc-gen-go"
       fallback: ""
+    - executable: protoc-gen-validate
+      version_min: ""
+      version_cmd: ""
+      artifact_url: "https://github.com/trpc-group/trpc-cmdline/releases/download/v0.0.1-${os}/protoc-gen-validate"
+      md5: ""
+      repository: "https://github.com/bufbuild/protoc-gen-validate/releases"
+      fallback: ""
+    - executable: protoc-gen-validate-go
+      version_min: ""
+      version_cmd: ""
+      artifact_url: "https://github.com/trpc-group/trpc-cmdline/releases/download/v0.0.1-${os}/protoc-gen-validate-go"
+      md5: ""
+      repository: "https://github.com/bufbuild/protoc-gen-validate/releases"
+      fallback: ""
     - executable: goimports
       version_min: ""
       version_cmd: ""