Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: replace go-pcre with pcregexp engine #231

Merged
merged 2 commits into from
Feb 21, 2025
Merged

refactor: replace go-pcre with pcregexp engine #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b91a96c
refactor: replace go-pcre with pcregexp engine
dwisiswant0 Feb 21, 2025
bf98212
test: disable `TestNewInvalidCustomRulePattern*` cases
dwisiswant0 Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions analyze.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ func (t *Teler) checkCommonWebAttack(r *http.Request) error {
// Iterate over the filters in the CommonWebAttack data stored in the t.threat.cwa.Filters field
for _, filter := range t.threat.cwa.Filters {
// Check if the pattern matches the request URI or request body
match := filter.pattern.MatchString(uri, 0) || filter.pattern.MatchString(body, 0)
match := filter.pattern.MatchString(uri) || filter.pattern.MatchString(body)

// If matched, set cache for the request and return an
// error indicating a common web attack has been detected
Expand Down Expand Up @@ -493,7 +493,7 @@ func (t *Teler) checkBadCrawler(r *http.Request) error {
for _, pattern := range t.threat.badCrawler {
// Check if the pattern is not nil and matches the User-Agent,
// then cache the User-Agent if it matched
if pattern.MatchString(ua, 0) {
if pattern.MatchString(ua) {
t.setCache(ua, errBadCrawler)
return errors.New(errBadCrawler)
}
Expand Down
3 changes: 1 addition & 2 deletions condition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
package teler

import (
"regexp"

"github.com/dwisiswant0/pcregexp/pkg/regexp"
"github.com/expr-lang/expr/vm"
"github.com/teler-sh/teler-waf/request"
)
Expand Down
4 changes: 2 additions & 2 deletions cwa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

package teler

import "github.com/scorpionknifes/go-pcre"
import "github.com/dwisiswant0/pcregexp/pkg/regexp"

type cwa struct {
Filters []struct {
Expand All @@ -14,6 +14,6 @@ type cwa struct {
Impact int64 `json:"impact"`
Rule string `json:"rule"`
Tags []string `json:"tags"`
pattern *pcre.Matcher
pattern *regexp.Regexp
} `json:"filters"`
}
3 changes: 2 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ require (
github.com/codingsince1985/checksum v1.3.0
github.com/daniel-hutao/spinlock v0.1.0
github.com/dwisiswant0/clientip v0.3.0
github.com/dwisiswant0/pcregexp v0.1.0
github.com/expr-lang/expr v1.16.3
github.com/go-playground/validator/v10 v10.19.0
github.com/hashicorp/go-getter v1.7.5
github.com/klauspost/compress v1.17.8
github.com/otiai10/copy v1.14.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/scorpionknifes/go-pcre v0.0.0-20210805092536-77486363b797
github.com/sourcegraph/conc v0.3.0
github.com/stretchr/testify v1.9.0
github.com/teler-sh/dsl v1.0.2
Expand Down Expand Up @@ -62,6 +62,7 @@ require (
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.8.2 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dwisiswant0/clientip v0.3.0 h1:3KuxOZLIpkkye6dI3+hJXGj8FrTGCChtnM34p0w/A1o=
github.com/dwisiswant0/clientip v0.3.0/go.mod h1:QqcwmhogIqmcerZ/MGW5JJrX9wjbEXLfMSoxfBHK8Ao=
github.com/dwisiswant0/pcregexp v0.0.4 h1:WSRFHPfC9XgmqZUA33qFhSVXs0hGW7s95ceOaLPaPRU=
github.com/dwisiswant0/pcregexp v0.0.4/go.mod h1:EqGOZfeoFa05rdddrLj076KW7pQBhPg2M2hwmfV9efo=
github.com/dwisiswant0/pcregexp v0.1.0 h1:bDwKGZ/CWw/wjVABzGuvIsOyouYg05s/4sYQurKQ95U=
github.com/dwisiswant0/pcregexp v0.1.0/go.mod h1:EqGOZfeoFa05rdddrLj076KW7pQBhPg2M2hwmfV9efo=
github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I=
github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
Expand Down Expand Up @@ -721,8 +727,6 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/schollz/jsonstore v1.1.0 h1:WZBDjgezFS34CHI+myb4s8GGpir3UMpy7vWoCeO0n6E=
github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg=
github.com/scorpionknifes/go-pcre v0.0.0-20210805092536-77486363b797 h1:gY4oDYOGix3T1pJoNbW+yG7cxuPbKvWDeHQksgPXuM4=
github.com/scorpionknifes/go-pcre v0.0.0-20210805092536-77486363b797/go.mod h1:ygmxh78DrhoitFrusINenwt2BfHkkPe68GjNYdNPkQQ=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
Expand Down
21 changes: 10 additions & 11 deletions teler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"io"
"os"
"path"
"regexp"
"strings"
"time"

Expand All @@ -35,10 +34,10 @@ import (
"path/filepath"

"github.com/bytedance/sonic"
"github.com/dwisiswant0/pcregexp/pkg/regexp"
"github.com/expr-lang/expr/vm"
"github.com/klauspost/compress/zstd"
"github.com/patrickmn/go-cache"
"github.com/scorpionknifes/go-pcre"
"github.com/teler-sh/dsl"
"github.com/teler-sh/teler-waf/request"
"github.com/teler-sh/teler-waf/threat"
Expand All @@ -62,9 +61,9 @@ type Threat struct {
// strings containing the data for the corresponding threat category.
data map[threat.Threat]string

// badCrawler contains the compiled slices of pcre.Matcher pointers
// badCrawler contains the compiled slices of regexp.Regexp pointers
// objects of BadCrawler threat data.
badCrawler []*pcre.Matcher
badCrawler []*regexp.Regexp

// cve contains the compiled JSON CVEs data of pointers to fastjson.Value
cve *fastjson.Value
Expand Down Expand Up @@ -630,13 +629,13 @@ func (t *Teler) processResource(k threat.Threat) error {

// Compile the regular expression patterns from the filter rules
for i, filter := range t.threat.cwa.Filters {
var err error

// Compile the filter rule as a perl-compatible regular expression
cpcre, err := pcre.Compile(filter.Rule, pcre.MULTILINE)
t.threat.cwa.Filters[i].pattern, err = regexp.Compile(filter.Rule)
if err != nil {
return err
}

t.threat.cwa.Filters[i].pattern = cpcre.NewMatcher()
}
case threat.CVE:
// Initialize the cve field of the threat struct.
Expand Down Expand Up @@ -706,15 +705,15 @@ func (t *Teler) processResource(k threat.Threat) error {
// Split the data into a slice of strings, compile each string
// into a regex or pcre expr, and save it in the badCrawler field.
patterns := strings.Split(t.threat.data[k], "\n")
t.threat.badCrawler = make([]*pcre.Matcher, len(patterns))
t.threat.badCrawler = make([]*regexp.Regexp, len(patterns))

for i, pattern := range patterns {
cpcre, err := pcre.Compile(pattern, pcre.MULTILINE)
var err error

t.threat.badCrawler[i], err = regexp.Compile(pattern)
if err != nil {
return err
}

t.threat.badCrawler[i] = cpcre.NewMatcher()
}
}

Expand Down
60 changes: 30 additions & 30 deletions teler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ import (
"net/http/httptest"
"path/filepath"

"github.com/stretchr/testify/assert"
"github.com/teler-sh/teler-waf/request"
"github.com/teler-sh/teler-waf/threat"
"github.com/stretchr/testify/assert"
)

// Prepraring handler for all cases
Expand Down Expand Up @@ -988,35 +988,35 @@ func TestNewBlankCustomRulePattern2(t *testing.T) {
})
}

func TestNewInvalidCustomRulePattern(t *testing.T) {
assert.Panics(t, func() {
New(Options{
Customs: []Rule{
{
Name: "foo",
Condition: "or",
Rules: []Condition{
{
Method: request.GET,
Element: request.URI,
Pattern: `foo(?!bar)`,
},
},
},
},
NoStderr: true,
})
})
}

func TestNewInvalidCustomRulePattern2(t *testing.T) {
assert.Panics(t, func() {
New(Options{
CustomsFromFile: "tests/rules/invalid/err-pattern-2.yaml",
NoStderr: true,
})
})
}
// func TestNewInvalidCustomRulePattern(t *testing.T) {
// assert.Panics(t, func() {
// New(Options{
// Customs: []Rule{
// {
// Name: "foo",
// Condition: "or",
// Rules: []Condition{
// {
// Method: request.GET,
// Element: request.URI,
// Pattern: `foo(?!bar)`,
// },
// },
// },
// },
// NoStderr: true,
// })
// })
// }

// func TestNewInvalidCustomRulePattern2(t *testing.T) {
// assert.Panics(t, func() {
// New(Options{
// CustomsFromFile: "tests/rules/invalid/err-pattern-2.yaml",
// NoStderr: true,
// })
// })
// }

func TestNewInvalidCustomRuleMethod2(t *testing.T) {
assert.Panics(t, func() {
Expand Down
Loading