From 80ede694dc3e0627ddbb8f3daaaa869adc4b470d Mon Sep 17 00:00:00 2001 From: Nicolas Carlier Date: Sat, 5 Nov 2022 10:50:36 +0000 Subject: [PATCH] feat(incoming-webhook): add scripting - remove rule engine from categories - remove notification control from categories - add script engine to incoming webhook --- README.md | 3 +- autogen/db/postgres/db_sql_migration.go | 7 + go.mod | 9 +- go.sum | 23 +-- pkg/constant/context.go | 2 +- pkg/db/postgres/category.go | 20 +-- pkg/db/postgres/incoming-webhooks.go | 9 +- pkg/db/postgres/migration.go | 2 +- pkg/db/postgres/sql/db_migration_13.sql | 5 + pkg/db/test/category_test.go | 27 +--- pkg/db/test/incoming-webhook_test.go | 11 +- pkg/event-listener/article-notification.go | 52 ++----- pkg/model/category.go | 35 +---- pkg/model/incoming-webhook.go | 17 ++- pkg/rule-engine/cache.go | 86 ----------- pkg/rule-engine/helper.go | 32 ---- pkg/rule-engine/processor.go | 116 -------------- pkg/rule-engine/test/processor_test.go | 142 ------------------ pkg/schema/category/mutations.go | 24 +-- pkg/schema/category/types.go | 29 ---- pkg/schema/incoming-webhook/mutations.go | 11 +- pkg/schema/incoming-webhook/types.go | 3 + pkg/scripting/engine.go | 50 ++++++ pkg/scripting/functions.go | 86 +++++++++++ pkg/scripting/interpreter.go | 65 ++++++++ pkg/scripting/operation.go | 34 +++++ pkg/scripting/test/engine_test.go | 35 +++++ pkg/scripting/test/interpreter_test.go | 77 ++++++++++ pkg/scripting/types.go | 10 ++ pkg/service/articles_create.go | 39 +++-- pkg/service/categories.go | 42 ------ pkg/service/registry.go | 6 +- pkg/service/rules.go | 45 ------ pkg/service/scripting.go | 70 +++++++++ pkg/service/test/articles_create_test.go | 19 --- pkg/service/test/articles_scripting_test.go | 43 ++++++ pkg/service/test/helper.go | 12 ++ scripts/payload.json | 9 +- .../articles/components/EditArticleForm.tsx | 4 +- ui/src/categories/models.ts | 2 - ui/src/categories/queries.ts | 10 +- ui/src/components/FormInputField.tsx | 2 +- ui/src/components/FormTextareaField.tsx | 3 +- .../settings/categories/AddCategoryForm.tsx | 24 +-- .../categories/CategoriesTab.module.css | 7 - ui/src/settings/categories/CategoriesTab.tsx | 15 -- .../settings/categories/EditCategoryForm.tsx | 23 +-- .../AddIncomingWebhookForm.tsx | 15 +- .../EditIncomingWebhookForm.tsx | 13 +- .../incoming-webhook/IncomingWebhookHelp.tsx | 5 +- .../intergrations/incoming-webhook/models.ts | 2 + .../intergrations/incoming-webhook/queries.ts | 7 +- .../AddOutgoingWebhookForm.tsx | 2 +- .../EditOutgoingWebhookForm.tsx | 2 +- 54 files changed, 653 insertions(+), 790 deletions(-) create mode 100644 pkg/db/postgres/sql/db_migration_13.sql delete mode 100644 pkg/rule-engine/cache.go delete mode 100644 pkg/rule-engine/helper.go delete mode 100644 pkg/rule-engine/processor.go delete mode 100644 pkg/rule-engine/test/processor_test.go create mode 100644 pkg/scripting/engine.go create mode 100644 pkg/scripting/functions.go create mode 100644 pkg/scripting/interpreter.go create mode 100644 pkg/scripting/operation.go create mode 100644 pkg/scripting/test/engine_test.go create mode 100644 pkg/scripting/test/interpreter_test.go create mode 100644 pkg/scripting/types.go delete mode 100644 pkg/service/rules.go create mode 100644 pkg/service/scripting.go create mode 100644 pkg/service/test/articles_scripting_test.go delete mode 100644 ui/src/settings/categories/CategoriesTab.module.css diff --git a/README.md b/README.md index 6683357b3..ca736a3df 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Read your Internet article flow in one place with complete peace of mind and fre - Read articles from anywhere in one place. - Save articles for offline reading or locally on you disk in the format you want (HTML, EPUB, ZIP, ...). -- Create categories and classify new articles automatically thanks to a customizable rule engine. +- Create categories and classify new articles. +- Customize article integration with a scripting engine. - Link with external services thanks to incoming and outgoing webhooks ([RSS][feedpushr], [Keeper][keeper], [Pocket][pocket], [Shaarli][shaarli], [Wallabag][wallabag], S3 bucket, and more...). - Receive notifications when new articles are available. - Enjoy the same user experience on mobile as on desktop thanks to [Progressive Web App][pwa] support. diff --git a/autogen/db/postgres/db_sql_migration.go b/autogen/db/postgres/db_sql_migration.go index 6844277de..8e2f5b28f 100644 --- a/autogen/db/postgres/db_sql_migration.go +++ b/autogen/db/postgres/db_sql_migration.go @@ -110,6 +110,12 @@ ALTER TYPE article_status ADD VALUE 'to_read' AFTER 'read'; alter table categories add column notification_strategy notification_strategy_type not null default 'none'; `, "db_migration_12": `alter table devices add column last_seen_at timestamp with time zone not null default now(); +`, + "db_migration_13": `alter table incoming_webhooks add column script varchar not null default 'return true;'; + +alter table categories drop column rule; +alter table categories drop column notification_strategy; +drop type notification_strategy_type; `, "db_migration_2": `create table devices ( id serial not null, @@ -167,6 +173,7 @@ var DatabaseSQLMigrationChecksums = map[string]string{ "db_migration_10": "935f7f7208d0230865d0915bf8f6b940331084d3aeb951536605f879a85a842f", "db_migration_11": "1150b8fa81099eb5956989560e8eebecafe5e39cbd1a5f6f7d23f3dfceb810bf", "db_migration_12": "b24497bb03f04fb4705ae752f8a5bf69dad26f168bc8ec196af93aee29deef49", + "db_migration_13": "4a52465eeb50a236d7f7a94cc51cd78238de0f885a6d29da4a548b5c389ebe81", "db_migration_2": "0be0d1ef1e9481d61db425a7d54378f3667c091949525b9c285b18660b6e8a1d", "db_migration_3": "5cd0d3628d990556c0b85739fd376c42244da7e98b66852b6411d27eda20c3fc", "db_migration_4": "d5fb83c15b523f15291310ff27d36c099c4ba68de2fd901c5ef5b70a18fedf65", diff --git a/go.mod b/go.mod index 994bc6970..12c478ac3 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/ncarlier/readflow require ( + github.com/BurntSushi/toml v1.0.0 github.com/Masterminds/squirrel v1.5.2 github.com/NYTimes/gziphandler v1.1.1 github.com/SherClockHolmes/webpush-go v1.1.3 - github.com/antonmedv/expr v1.9.0 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef github.com/bits-and-blooms/bloom/v3 v3.1.0 github.com/brianvoe/gofakeit v3.18.0+incompatible @@ -13,9 +13,11 @@ require ( github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 github.com/go-shiori/go-readability v0.0.0-20210627123243-82cc33435520 github.com/golang-jwt/jwt/v4 v4.2.0 + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/uuid v1.3.0 github.com/graphql-go/graphql v0.8.0 github.com/graphql-go/handler v0.2.3 + github.com/imdario/mergo v0.3.12 github.com/lib/pq v1.10.4 github.com/microcosm-cc/bluemonday v1.0.16 github.com/minio/minio-go/v7 v7.0.18 @@ -23,6 +25,7 @@ require ( github.com/rs/zerolog v1.26.1 github.com/sethvargo/go-limiter v0.7.2 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e + github.com/skx/evalfilter/v2 v2.1.19 github.com/speps/go-hashids/v2 v2.0.1 github.com/stretchr/testify v1.7.0 go.etcd.io/bbolt v1.3.6 @@ -31,7 +34,6 @@ require ( ) require ( - github.com/BurntSushi/toml v1.0.0 github.com/andybalholm/cascadia v1.2.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -41,10 +43,9 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.4.3 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/imdario/mergo v0.3.12 github.com/json-iterator/go v1.1.11 // indirect github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect diff --git a/go.sum b/go.sum index 7150082dd..f02de4cbd 100644 --- a/go.sum +++ b/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Masterminds/squirrel v1.5.2 h1:UiOEi2ZX4RCSkpiNDQN5kro/XIBpSRk9iTqdIRPzUXE= github.com/Masterminds/squirrel v1.5.2/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= @@ -18,8 +17,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= -github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU= -github.com/antonmedv/expr v1.9.0/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= @@ -49,7 +46,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -59,8 +55,6 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= -github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -86,6 +80,8 @@ github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -157,11 +153,7 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhR github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= -github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc= @@ -189,7 +181,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -215,8 +206,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= @@ -224,7 +213,6 @@ github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-limiter v0.7.2 h1:FgC4N7RMpV5gMrUdda15FaFTkQ/L4fEqM7seXMs4oO8= @@ -237,6 +225,9 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= +github.com/skx/evalfilter/v2 v2.1.19 h1:EeoWfXVi1bw1FeBlojIHrZsU7YVOFCj9tQIj6u6aPRQ= +github.com/skx/evalfilter/v2 v2.1.19/go.mod h1:Kett8pWXFLb9kfeytJiO+/Ci534U8YcNp6WD+2goYsI= +github.com/skx/subcommands v0.9.1/go.mod h1:HpOZHVUXT5Rc/Q7UCiyj7h5u6BleDfFjt+vxy2igonA= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -256,11 +247,9 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -321,10 +310,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/constant/context.go b/pkg/constant/context.go index 8c4faea90..c990a2521 100644 --- a/pkg/constant/context.go +++ b/pkg/constant/context.go @@ -9,7 +9,7 @@ const ( ContextUser // ContextRequestID is the key used to store request ID into the request context ContextRequestID - // ContextIncomingWebhookAlias is the key used to store incomimg webhook alias into the request context + // ContextIncomingWebhookAlias is the key used to store incoming webhook alias into the request context ContextIncomingWebhookAlias // ContextIsAdmin is true when the user is an admin ContextIsAdmin diff --git a/pkg/db/postgres/category.go b/pkg/db/postgres/category.go index e516be8f0..cab72e7c3 100644 --- a/pkg/db/postgres/category.go +++ b/pkg/db/postgres/category.go @@ -13,8 +13,6 @@ var categoryColumns = []string{ "id", "user_id", "title", - "rule", - "notification_strategy", "created_at", "updated_at", } @@ -26,8 +24,6 @@ func mapRowToCategory(row *sql.Row) (*model.Category, error) { &cat.ID, &cat.UserID, &cat.Title, - &cat.Rule, - &cat.NotificationStrategy, &cat.CreatedAt, &cat.UpdatedAt, ) @@ -44,12 +40,10 @@ func (pg *DB) CreateCategoryForUser(uid uint, form model.CategoryCreateForm) (*m query, args, _ := pg.psql.Insert( "categories", ).Columns( - "user_id", "title", "rule", "notification_strategy", + "user_id", "title", ).Values( uid, form.Title, - form.Rule, - form.NotificationStrategy, ).Suffix( "RETURNING " + strings.Join(categoryColumns, ","), ).ToSql() @@ -65,16 +59,6 @@ func (pg *DB) UpdateCategoryForUser(uid uint, form model.CategoryUpdateForm) (*m if form.Title != nil { update["title"] = *form.Title } - if form.Rule != nil { - if *form.Rule == "" { - update["rule"] = nil - } else { - update["rule"] = *form.Rule - } - } - if form.NotificationStrategy != nil { - update["notification_strategy"] = *form.NotificationStrategy - } query, args, _ := pg.psql.Update( "categories", ).SetMap(update).Where( @@ -135,8 +119,6 @@ func (pg *DB) GetCategoriesByUser(uid uint) ([]model.Category, error) { &cat.ID, &cat.UserID, &cat.Title, - &cat.Rule, - &cat.NotificationStrategy, &cat.CreatedAt, &cat.UpdatedAt, ) diff --git a/pkg/db/postgres/incoming-webhooks.go b/pkg/db/postgres/incoming-webhooks.go index a45c8777f..4f2edd9ad 100644 --- a/pkg/db/postgres/incoming-webhooks.go +++ b/pkg/db/postgres/incoming-webhooks.go @@ -14,6 +14,7 @@ var inboundServiceColumns = []string{ "user_id", "alias", "token", + "script", "last_usage_at", "created_at", "updated_at", @@ -27,6 +28,7 @@ func mapRowToIncomingWebhook(row *sql.Row) (*model.IncomingWebhook, error) { &result.UserID, &result.Alias, &result.Token, + &result.Script, &result.LastUsageAt, &result.CreatedAt, &result.UpdatedAt, @@ -45,11 +47,12 @@ func (pg *DB) CreateIncomingWebhookForUser(uid uint, form model.IncomingWebhookC query, args, _ := pg.psql.Insert( "incoming_webhooks", ).Columns( - "user_id", "alias", "token", + "user_id", "alias", "token", "script", ).Values( uid, form.Alias, form.Token, + form.Script, ).Suffix( "RETURNING " + strings.Join(inboundServiceColumns, ","), ).ToSql() @@ -65,6 +68,9 @@ func (pg *DB) UpdateIncomingWebhookForUser(uid uint, form model.IncomingWebhookU if form.Alias != nil { update["alias"] = *form.Alias } + if form.Script != nil { + update["script"] = *form.Script + } query, args, _ := pg.psql.Update( "incoming_webhooks", ).SetMap(update).Where( @@ -140,6 +146,7 @@ func (pg *DB) GetIncomingWebhooksByUser(uid uint) ([]model.IncomingWebhook, erro &item.UserID, &item.Alias, &item.Token, + &item.Script, &item.LastUsageAt, &item.CreatedAt, &item.UpdatedAt, diff --git a/pkg/db/postgres/migration.go b/pkg/db/postgres/migration.go index 6e1101b80..207291f40 100644 --- a/pkg/db/postgres/migration.go +++ b/pkg/db/postgres/migration.go @@ -8,7 +8,7 @@ import ( "github.com/rs/zerolog/log" ) -const schemaVersion = 12 +const schemaVersion = 13 // Migrate executes database migrations. func Migrate(db *sql.DB) { diff --git a/pkg/db/postgres/sql/db_migration_13.sql b/pkg/db/postgres/sql/db_migration_13.sql new file mode 100644 index 000000000..8099b0779 --- /dev/null +++ b/pkg/db/postgres/sql/db_migration_13.sql @@ -0,0 +1,5 @@ +alter table incoming_webhooks add column script varchar not null default 'return true;'; + +alter table categories drop column rule; +alter table categories drop column notification_strategy; +drop type notification_strategy_type; diff --git a/pkg/db/test/category_test.go b/pkg/db/test/category_test.go index da77f4a09..6eeb16b60 100644 --- a/pkg/db/test/category_test.go +++ b/pkg/db/test/category_test.go @@ -16,8 +16,7 @@ func assertCategoryExists(t *testing.T, uid uint, title string, notif string) *m } createForm := model.CategoryCreateForm{ - Title: title, - NotificationStrategy: notif, + Title: title, } category, err = testDB.CreateCategoryForUser(uid, createForm) @@ -25,8 +24,6 @@ func assertCategoryExists(t *testing.T, uid uint, title string, notif string) *m assert.NotNil(t, category) assert.NotNil(t, category.ID) assert.Equal(t, title, category.Title) - assert.Equal(t, notif, category.NotificationStrategy) - assert.Nil(t, category.Rule) return category } func TestCreateAndUpdateCategory(t *testing.T) { @@ -40,33 +37,15 @@ func TestCreateAndUpdateCategory(t *testing.T) { // Update category title title = "My updated category" - notif := "global" update := model.CategoryUpdateForm{ - ID: *category.ID, - Title: &title, - NotificationStrategy: ¬if, + ID: *category.ID, + Title: &title, } category, err := testDB.UpdateCategoryForUser(uid, update) assert.Nil(t, err) assert.NotNil(t, category) assert.NotNil(t, category.ID) assert.Equal(t, title, category.Title) - assert.Nil(t, category.Rule) - assert.Equal(t, notif, category.NotificationStrategy) - - // Update category title - rule := "title matches \"test\"" - update = model.CategoryUpdateForm{ - ID: *category.ID, - Rule: &rule, - } - category, err = testDB.UpdateCategoryForUser(uid, update) - assert.Nil(t, err) - assert.NotNil(t, category) - assert.NotNil(t, category.ID) - assert.Equal(t, title, category.Title) - assert.Equal(t, rule, *category.Rule) - assert.Equal(t, notif, category.NotificationStrategy) // Count categories of test user nb, err := testDB.CountCategoriesByUser(uid) diff --git a/pkg/db/test/incoming-webhook_test.go b/pkg/db/test/incoming-webhook_test.go index 1a1737fdb..7142808ab 100644 --- a/pkg/db/test/incoming-webhook_test.go +++ b/pkg/db/test/incoming-webhook_test.go @@ -8,7 +8,7 @@ import ( "github.com/ncarlier/readflow/pkg/model" ) -func assertIncomingWebhookExists(t *testing.T, uid uint, alias string) *model.IncomingWebhook { +func assertIncomingWebhookExists(t *testing.T, uid uint, alias string, script string) *model.IncomingWebhook { webhook, err := testDB.GetIncomingWebhookByUserAndAlias(uid, alias) assert.Nil(t, err) if webhook != nil { @@ -16,13 +16,14 @@ func assertIncomingWebhookExists(t *testing.T, uid uint, alias string) *model.In } builder := model.NewIncomingWebhookCreateFormBuilder() - form := builder.Alias(alias).Build() + form := builder.Alias(alias).Script(script).Build() webhook, err = testDB.CreateIncomingWebhookForUser(uid, *form) assert.Nil(t, err) assert.NotNil(t, webhook) assert.NotNil(t, webhook.ID) assert.Equal(t, alias, webhook.Alias) + assert.Equal(t, script, webhook.Script) assert.NotEqual(t, "", webhook.Token) return webhook } @@ -32,8 +33,9 @@ func TestCreateOrUpdateIncomingWebhook(t *testing.T) { uid := *testUser.ID alias := "My test incoming webhook" + script := "return false;" - assertIncomingWebhookExists(t, uid, alias) + assertIncomingWebhookExists(t, uid, alias, script) } func TestDeleteIncomingWebhook(t *testing.T) { @@ -42,9 +44,10 @@ func TestDeleteIncomingWebhook(t *testing.T) { uid := *testUser.ID alias := "My incoming webhook" + script := "return false;" // Assert webhook exists - webhook := assertIncomingWebhookExists(t, uid, alias) + webhook := assertIncomingWebhookExists(t, uid, alias, script) err := testDB.DeleteIncomingWebhookByUser(uid, *webhook.ID) assert.Nil(t, err) diff --git a/pkg/event-listener/article-notification.go b/pkg/event-listener/article-notification.go index 074afeceb..3a871437a 100644 --- a/pkg/event-listener/article-notification.go +++ b/pkg/event-listener/article-notification.go @@ -45,49 +45,29 @@ func init() { return } - globalStrategy := true - text := "You have a new article to read." - href := "/" - if article.CategoryID != nil { - category, err := service.Lookup().GetCategory(ctx, *article.CategoryID) - if err != nil { - log.Info().Err(err).Uint("id", uid).Msg(errorMessage) - return - } - if category.NotificationStrategy == "none" { - return - } - globalStrategy = category.NotificationStrategy == "global" - text = fmt.Sprintf("You have a new article to read in %s", category.Title) - href = fmt.Sprintf("/categories/%d/%d", *category.ID, article.ID) + // Send notification only if user is inactive for a while + lastLoginDelay := time.Now().Add(-maxUserInactivityBeforeNotification) + if user.Enabled && user.LastLoginAt != nil && user.LastLoginAt.After(lastLoginDelay) { + return } - - if globalStrategy { - // Send notification only if user is inactive for a while - lastLoginDelay := time.Now().Add(-maxUserInactivityBeforeNotification) - if user.Enabled && user.LastLoginAt != nil && user.LastLoginAt.After(lastLoginDelay) { - return - } - // Retrieve number of articles - nb, err := service.Lookup().CountCurrentUserArticles(ctx, req) - if err != nil { - log.Info().Err(err).Uint("id", uid).Msg(errorMessage) - return - } - // Send notification only every 10 articles - if !(nb > 0 && math.Mod(float64(nb), 10) == 0) { - return - } - // Format text message - text = fmt.Sprintf("You have %d articles to read.", nb) - href = "/" + // Retrieve number of articles + nb, err = service.Lookup().CountCurrentUserArticles(ctx, req) + if err != nil { + log.Info().Err(err).Uint("id", uid).Msg(errorMessage) + return + } + // Send notification only every 10 articles + if !(nb > 0 && math.Mod(float64(nb), 10) == 0) { + return } + text := fmt.Sprintf("You have %d articles to read.", nb) + // Build notification notif := &model.DeviceNotification{ Title: "New articles to read", Body: text, - Href: href, + Href: "/", } b, err := json.Marshal(notif) if err == nil { diff --git a/pkg/model/category.go b/pkg/model/category.go index 531725a2f..bf0b4971f 100644 --- a/pkg/model/category.go +++ b/pkg/model/category.go @@ -8,28 +8,22 @@ import ( // Category structure definition type Category struct { - ID *uint `json:"id,omitempty"` - UserID *uint `json:"user_id,omitempty"` - Title string `json:"title,omitempty"` - Rule *string `json:"rule,omitempty"` - NotificationStrategy string `json:"notification_strategy,omitempty"` - CreatedAt *time.Time `json:"created_at,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` + ID *uint `json:"id,omitempty"` + UserID *uint `json:"user_id,omitempty"` + Title string `json:"title,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` } // CategoryCreateForm structure definition type CategoryCreateForm struct { - Title string - Rule *string - NotificationStrategy string + Title string } // CategoryUpdateForm structure definition type CategoryUpdateForm struct { - ID uint - Title *string - Rule *string - NotificationStrategy *string + ID uint + Title *string } // CategoryCreateFormBuilder is a builder to create an CategoryCreateForm @@ -52,18 +46,5 @@ func (cb *CategoryCreateFormBuilder) Build() *CategoryCreateForm { func (cb *CategoryCreateFormBuilder) Random() *CategoryCreateFormBuilder { gofakeit.Seed(0) cb.form.Title = gofakeit.Word() - cb.form.NotificationStrategy = gofakeit.RandString([]string{"none", "global", "individual"}) - return cb -} - -// Rule set category rule -func (cb *CategoryCreateFormBuilder) Rule(rule *string) *CategoryCreateFormBuilder { - cb.form.Rule = rule - return cb -} - -// Notification set incoming webhook notification -func (cb *CategoryCreateFormBuilder) NotificationStrategy(strategy string) *CategoryCreateFormBuilder { - cb.form.NotificationStrategy = strategy return cb } diff --git a/pkg/model/incoming-webhook.go b/pkg/model/incoming-webhook.go index 022cb3140..a0fa329f2 100644 --- a/pkg/model/incoming-webhook.go +++ b/pkg/model/incoming-webhook.go @@ -12,6 +12,7 @@ type IncomingWebhook struct { UserID uint `json:"user_id,omitempty"` Alias string `json:"alias,omitempty"` Token string `json:"token,omitempty"` + Script string `json:"script,omitempty"` LastUsageAt *time.Time `json:"last_usage_at,omitempty"` CreatedAt *time.Time `json:"created_at,omitempty"` UpdatedAt *time.Time `json:"updated_at,omitempty"` @@ -19,14 +20,16 @@ type IncomingWebhook struct { // IncomingWebhookCreateForm structure definition type IncomingWebhookCreateForm struct { - Alias string - Token string + Alias string + Token string + Script string } // IncomingWebhookUpdateForm structure definition type IncomingWebhookUpdateForm struct { - ID uint - Alias *string + ID uint + Alias *string + Script *string } // IncomingWebhookCreateFormBuilder is a builder to create an incoming webhook create form @@ -51,3 +54,9 @@ func (ab *IncomingWebhookCreateFormBuilder) Alias(alias string) *IncomingWebhook ab.form.Alias = alias return ab } + +// Script set incoming webhook script +func (ab *IncomingWebhookCreateFormBuilder) Script(script string) *IncomingWebhookCreateFormBuilder { + ab.form.Script = script + return ab +} diff --git a/pkg/rule-engine/cache.go b/pkg/rule-engine/cache.go deleted file mode 100644 index b5b672186..000000000 --- a/pkg/rule-engine/cache.go +++ /dev/null @@ -1,86 +0,0 @@ -package ruleengine - -import ( - "container/list" - "sync" -) - -// Cache is a cache to strore rules processors -// LRU strategy: evicts least recently used items -type Cache struct { - capacity int - items map[uint]*CacheItem - list *list.List - lock *sync.RWMutex -} - -// CacheItem is an item of the cache -type CacheItem struct { - key uint - value *ProcessorPipeline - listElement *list.Element -} - -// NewRuleEngineCache creates a new cache for the rule engine -func NewRuleEngineCache(capacity int) *Cache { - return &Cache{ - capacity: capacity, - items: make(map[uint]*CacheItem, capacity), - list: list.New(), - lock: new(sync.RWMutex), - } -} - -// Get retrieve a list of rule processor from the cache -func (c *Cache) Get(key uint) *ProcessorPipeline { - c.lock.RLock() - defer c.lock.RUnlock() - if item, exists := c.items[key]; exists { - c.promote(item) - return item.value - } - return nil -} - -// Set put a list of rule processor into the cache -func (c *Cache) Set(key uint, value *ProcessorPipeline) { - c.lock.Lock() - defer c.lock.Unlock() - if c.capacity <= c.list.Len() { - c.prune() - } - - if item, exists := c.items[key]; exists { - item.value = value - c.promote(item) - } else { - item = &CacheItem{key: key, value: value} - item.listElement = c.list.PushFront(item) - c.items[key] = item - } -} - -// Evict remove a list of rule processor from the cache -func (c *Cache) Evict(key uint) { - c.lock.RLock() - defer c.lock.RUnlock() - if item, exists := c.items[key]; exists { - c.list.Remove(item.listElement) - delete(c.items, item.key) - } -} - -func (c *Cache) promote(item *CacheItem) { - c.list.MoveToFront(item.listElement) -} - -func (c *Cache) prune() { - for i := 0; i < 50; i++ { - tail := c.list.Back() - if tail == nil { - return - } - item := c.list.Remove(tail).(*CacheItem) - delete(c.items, item.key) - } -} diff --git a/pkg/rule-engine/helper.go b/pkg/rule-engine/helper.go deleted file mode 100644 index 0674812ef..000000000 --- a/pkg/rule-engine/helper.go +++ /dev/null @@ -1,32 +0,0 @@ -package ruleengine - -func toBool(i1 interface{}) bool { - if i1 == nil { - return false - } - switch i2 := i1.(type) { - default: - return false - case bool: - return i2 - case string: - return i2 == "true" - case int: - return i2 != 0 - case *bool: - if i2 == nil { - return false - } - return *i2 - case *string: - if i2 == nil { - return false - } - return *i2 == "true" - case *int: - if i2 == nil { - return false - } - return *i2 != 0 - } -} diff --git a/pkg/rule-engine/processor.go b/pkg/rule-engine/processor.go deleted file mode 100644 index a7162345c..000000000 --- a/pkg/rule-engine/processor.go +++ /dev/null @@ -1,116 +0,0 @@ -package ruleengine - -import ( - "context" - "fmt" - "strings" - - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - - "github.com/ncarlier/readflow/pkg/constant" - "github.com/ncarlier/readflow/pkg/model" -) - -// RuleProcessor define a rule processor -type RuleProcessor struct { - category model.Category - program *vm.Program -} - -// NewRuleProcessor creates a new rule processor -func NewRuleProcessor(category model.Category) (*RuleProcessor, error) { - if category.Rule == nil || *category.Rule == "" { - return nil, nil - } - env := map[string]interface{}{ - "title": "", - "text": "", - "url": "", - "webhook": "", - "tags": []string{}, - "toLower": strings.ToLower, - "toUpper": strings.ToUpper, - } - p, err := expr.Compile(*category.Rule, expr.Env(env)) - if err != nil { - return nil, fmt.Errorf("invalid rule expression: %s", err) - } - return &RuleProcessor{ - category: category, - program: p, - }, nil -} - -// Apply a rule on an article -func (rp *RuleProcessor) Apply(ctx context.Context, article *model.ArticleCreateForm) (bool, error) { - tags := []string{} - if article.Tags != nil { - tags = strings.Split(*article.Tags, ",") - } - text := "" - if article.Text != nil { - text = *article.Text - } - url := "" - if article.URL != nil { - url = *article.URL - } - incomingWebhookAlias := "" - if alias := ctx.Value(constant.ContextIncomingWebhookAlias); alias != nil { - incomingWebhookAlias = alias.(string) - } - - env := map[string]interface{}{ - "title": article.Title, - "text": text, - "url": url, - "webhook": incomingWebhookAlias, - "tags": tags, - "toLower": strings.ToLower, - "toUpper": strings.ToUpper, - } - - result, err := expr.Run(rp.program, env) - if err != nil { - return false, fmt.Errorf("unable to eval expression on article %s: %s", *article.URL, err) - } - match := toBool(result) - if match { - article.CategoryID = rp.category.ID - } - - return match, nil -} - -// ProcessorPipeline is a list of rule processor -type ProcessorPipeline []*RuleProcessor - -// NewProcessorsPipeline creates a processor pipeline from a category set -func NewProcessorsPipeline(categories []model.Category) (*ProcessorPipeline, error) { - result := ProcessorPipeline{} - for _, category := range categories { - processor, err := NewRuleProcessor(category) - if err != nil { - return nil, err - } - if processor != nil { - result = append(result, processor) - } - } - return &result, nil -} - -// Apply a processor pipeline on an article -func (pp *ProcessorPipeline) Apply(ctx context.Context, article *model.ArticleCreateForm) (bool, error) { - for _, processor := range *pp { - applied, err := processor.Apply(ctx, article) - if err != nil { - return false, err - } - if applied { - return true, nil - } - } - return false, nil -} diff --git a/pkg/rule-engine/test/processor_test.go b/pkg/rule-engine/test/processor_test.go deleted file mode 100644 index aa6d82cbe..000000000 --- a/pkg/rule-engine/test/processor_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package test - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/ncarlier/readflow/pkg/constant" - "github.com/ncarlier/readflow/pkg/model" - ruleengine "github.com/ncarlier/readflow/pkg/rule-engine" -) - -func newTestCategory(id uint, rule string) model.Category { - return model.Category{ - ID: &id, - Title: "Dummy category", - Rule: &rule, - } -} - -type testCase struct { - article *model.ArticleCreateForm - category model.Category - expectedValue *uint - expectedError error -} - -var testCases = []testCase{} - -func init() { - expectedCategoryID := uint(42) - builder := model.NewArticleCreateFormBuilder() - testCases = append(testCases, testCase{ - article: builder.Random().Build(), - category: newTestCategory(42, "."), - expectedValue: nil, - expectedError: errors.New("invalid rule expression:"), - }, testCase{ - article: builder.Random().Title("foo").Build(), - category: newTestCategory(42, "title == \"foo\""), - expectedValue: &expectedCategoryID, - expectedError: nil, - }, testCase{ - article: builder.Random().Text("bar foo bar").Build(), - category: newTestCategory(42, "text matches \"foo\""), - expectedValue: &expectedCategoryID, - expectedError: nil, - }, testCase{ - article: builder.Random().Text("bar bar bar").Build(), - category: newTestCategory(42, "text matches \"foo\""), - expectedValue: nil, - expectedError: nil, - }, testCase{ - article: builder.Random().Tags("foo,bar").Build(), - category: newTestCategory(42, "\"foo\" in tags"), - expectedValue: &expectedCategoryID, - expectedError: nil, - }, testCase{ - article: builder.Random().Tags("foo,bar").Build(), - category: newTestCategory(42, "\"test\" not in tags"), - expectedValue: &expectedCategoryID, - expectedError: nil, - }, testCase{ - article: builder.Random().Title("Foo or Bar").Build(), - category: newTestCategory(42, "toLower(title) contains \"bar\""), - expectedValue: &expectedCategoryID, - expectedError: nil, - }) -} - -func TestRulesTestCases(t *testing.T) { - ctx := context.TODO() - - for idx, tc := range testCases { - testCaseName := fmt.Sprintf("Test case #%d", idx) - processor, err := ruleengine.NewRuleProcessor(tc.category) - if tc.expectedError != nil { - assert.NotNil(t, err, testCaseName) - assert.True(t, strings.HasPrefix(err.Error(), tc.expectedError.Error()), testCaseName) - assert.Nil(t, processor, testCaseName) - continue - } - assert.Nil(t, err, testCaseName) - assert.NotNil(t, processor, testCaseName) - applied, err := processor.Apply(ctx, tc.article) - assert.Nil(t, err, testCaseName) - if tc.expectedValue == nil { - assert.False(t, applied, testCaseName) - } else { - assert.True(t, applied, testCaseName) - assert.NotNil(t, tc.article.CategoryID, testCaseName) - assert.Equal(t, *tc.expectedValue, *tc.article.CategoryID, testCaseName) - } - } -} - -func TestProcessorPipeline(t *testing.T) { - ctx := context.TODO() - categories := []model.Category{ - newTestCategory(1, "title == \"test\""), - newTestCategory(2, "text matches \"foo\""), - newTestCategory(3, "\"foo\" in tags"), - } - pipeline, err := ruleengine.NewProcessorsPipeline(categories) - assert.Nil(t, err) - assert.NotNil(t, pipeline) - - builder := model.NewArticleCreateFormBuilder() - article := builder.Random().Text("foo bar foo").Build() - applied, err := pipeline.Apply(ctx, article) - assert.Nil(t, err) - assert.True(t, applied) - assert.NotNil(t, article.CategoryID) - assert.Equal(t, uint(2), *article.CategoryID) - - builder = model.NewArticleCreateFormBuilder() - article = builder.Random().Text("other").Build() - applied, err = pipeline.Apply(ctx, article) - assert.Nil(t, err) - assert.False(t, applied) - assert.Nil(t, article.CategoryID) -} - -func TestRuleProcessorWithContext(t *testing.T) { - ctx := context.WithValue(context.TODO(), constant.ContextIncomingWebhookAlias, "test") - category := newTestCategory(9, "webhook == \"test\"") - processor, err := ruleengine.NewRuleProcessor(category) - assert.Nil(t, err) - assert.NotNil(t, processor) - - builder := model.NewArticleCreateFormBuilder() - article := builder.Random().Title("test").Build() - applied, err := processor.Apply(ctx, article) - assert.Nil(t, err, "error should be nil") - assert.True(t, applied, "processor should be applied") - assert.True(t, article.CategoryID != nil, "category should be not nil") - assert.Equal(t, uint(9), *article.CategoryID, "category should be updated") -} diff --git a/pkg/schema/category/mutations.go b/pkg/schema/category/mutations.go index 573f8501a..94174fd26 100644 --- a/pkg/schema/category/mutations.go +++ b/pkg/schema/category/mutations.go @@ -22,42 +22,24 @@ var createOrUpdateCategoryMutationField = &graphql.Field{ Type: graphql.String, Description: "title of the category", }, - "rule": &graphql.ArgumentConfig{ - Type: graphql.String, - Description: "rule definition to put articles into this category", - }, - "notification_strategy": &graphql.ArgumentConfig{ - Type: notificationStrategy, - Description: "notification strategy for this category (by default: none)", - }, }, Resolve: createOrUpdateCategoryResolver, } func createOrUpdateCategoryResolver(p graphql.ResolveParams) (interface{}, error) { title := helper.GetGQLStringParameter("title", p.Args) - rule := helper.GetGQLStringParameter("rule", p.Args) - notif := helper.GetGQLStringParameter("notification_strategy", p.Args) if id, ok := helper.ConvGQLStringToUint(p.Args["id"]); ok { form := model.CategoryUpdateForm{ - ID: id, - Title: title, - Rule: rule, - NotificationStrategy: notif, + ID: id, + Title: title, } return service.Lookup().UpdateCategory(p.Context, form) } if title == nil { return nil, errors.New("title is required when creating a new category") } - if notif == nil { - defaultNotificationStrategy := "none" - notif = &defaultNotificationStrategy - } form := model.CategoryCreateForm{ - Title: *title, - Rule: rule, - NotificationStrategy: *notif, + Title: *title, } return service.Lookup().CreateCategory(p.Context, form) } diff --git a/pkg/schema/category/types.go b/pkg/schema/category/types.go index e7b2890a8..dc422d014 100644 --- a/pkg/schema/category/types.go +++ b/pkg/schema/category/types.go @@ -8,27 +8,6 @@ import ( "github.com/ncarlier/readflow/pkg/service" ) -var notificationStrategy = graphql.NewEnum( - graphql.EnumConfig{ - Name: "notificationStrategy", - Description: "Notification strategy", - Values: graphql.EnumValueConfigMap{ - "none": &graphql.EnumValueConfig{ - Value: "none", - Description: "no notification will be sent", - }, - "individual": &graphql.EnumValueConfig{ - Value: "individual", - Description: "a notification will be sent as soon as an article is received", - }, - "global": &graphql.EnumValueConfig{ - Value: "global", - Description: "a notification will be sent using the global strategy", - }, - }, - }, -) - // Type of a category var Type = graphql.NewObject( graphql.ObjectConfig{ @@ -41,14 +20,6 @@ var Type = graphql.NewObject( Type: graphql.String, Description: "title of the category", }, - "rule": &graphql.Field{ - Type: graphql.String, - Description: "rule definition to put articles into this category", - }, - "notification_strategy": &graphql.Field{ - Type: notificationStrategy, - Description: "notification strategy for this category", - }, "inbox": &graphql.Field{ Type: graphql.Int, Description: "number of received articles for this category", diff --git a/pkg/schema/incoming-webhook/mutations.go b/pkg/schema/incoming-webhook/mutations.go index ec673f59a..870498d8a 100644 --- a/pkg/schema/incoming-webhook/mutations.go +++ b/pkg/schema/incoming-webhook/mutations.go @@ -21,21 +21,26 @@ var createOrUpdateIncomingWebhookMutationField = &graphql.Field{ "alias": &graphql.ArgumentConfig{ Type: graphql.NewNonNull(graphql.String), }, + "script": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, }, Resolve: createOrUpdateIncomingWebhookResolver, } func createOrUpdateIncomingWebhookResolver(p graphql.ResolveParams) (interface{}, error) { alias := helper.GetGQLStringParameter("alias", p.Args) + script := helper.GetGQLStringParameter("script", p.Args) if id, ok := helper.ConvGQLStringToUint(p.Args["id"]); ok { form := model.IncomingWebhookUpdateForm{ - ID: id, - Alias: alias, + ID: id, + Alias: alias, + Script: script, } return service.Lookup().UpdateIncomingWebhook(p.Context, form) } builder := model.NewIncomingWebhookCreateFormBuilder() - form := builder.Alias(*alias).Build() + form := builder.Alias(*alias).Script(*script).Build() return service.Lookup().CreateIncomingWebhook(p.Context, *form) } diff --git a/pkg/schema/incoming-webhook/types.go b/pkg/schema/incoming-webhook/types.go index a93a4e73c..9e58fe386 100644 --- a/pkg/schema/incoming-webhook/types.go +++ b/pkg/schema/incoming-webhook/types.go @@ -17,6 +17,9 @@ var incomingWebhookType = graphql.NewObject( "token": &graphql.Field{ Type: graphql.String, }, + "script": &graphql.Field{ + Type: graphql.String, + }, "last_usage_at": &graphql.Field{ Type: graphql.DateTime, }, diff --git a/pkg/scripting/engine.go b/pkg/scripting/engine.go new file mode 100644 index 000000000..078ab8669 --- /dev/null +++ b/pkg/scripting/engine.go @@ -0,0 +1,50 @@ +package scripting + +import ( + "context" + "fmt" + + "github.com/golang/groupcache/lru" + "github.com/ncarlier/readflow/pkg/helper" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// ScriptEngine is the scripting engine +type ScriptEngine struct { + cache *lru.Cache + logger zerolog.Logger +} + +// NewScriptEngine create new script engine +func NewScriptEngine(cacheSize int) *ScriptEngine { + logger := log.With().Str("component", "scripting-engine").Logger() + return &ScriptEngine{ + cache: lru.New(cacheSize), + logger: logger, + } +} + +// Exec a script by the script engine +func (s *ScriptEngine) Exec(ctx context.Context, script string, input ScriptInput) (OperationStack, error) { + key := helper.Hash(script) + logger := s.logger.With().Str("key", key).Str("title", input.Title).Logger() + cacheItem, ok := s.cache.Get(key) + var interpreter *Interpreter + if ok { + var valid bool + if interpreter, valid = cacheItem.(*Interpreter); !valid { + return nil, fmt.Errorf("script engine cache is poisoned") + } + logger.Debug().Msg("interpreter loaded from cache") + } else { + var err error + logger.Debug().Msg("creating interpreter...") + if interpreter, err = NewInterpreter(script, s.logger); err != nil { + return nil, err + } + s.cache.Add(key, interpreter) + } + logger.Debug().Msg("executing...") + return interpreter.Exec(ctx, input) +} diff --git a/pkg/scripting/functions.go b/pkg/scripting/functions.go new file mode 100644 index 000000000..d84d39d85 --- /dev/null +++ b/pkg/scripting/functions.go @@ -0,0 +1,86 @@ +package scripting + +import ( + "fmt" + + "github.com/skx/evalfilter/v2/object" +) + +// fnNoOp do nothing +func fnNoOp(args []object.Object) object.Object { + return &object.Void{} +} + +// fnPrint is the implementation of our `print` function. +func (i *Interpreter) fnPrint(args []object.Object) object.Object { + for _, e := range args { + i.logger.Debug().Str("fn", "print").Msg(e.Inspect()) + } + return &object.Void{} +} + +// fnPrintf is the implementation of our `printf` function. +func (i *Interpreter) fnPrintf(args []object.Object) object.Object { + // We expect 1+ arguments + if len(args) < 1 { + return &object.Null{} + } + // Type-check + if args[0].Type() != object.STRING { + return &object.Null{} + } + // Get the format-string. + fs := args[0].(*object.String).Value + // Convert the arguments to something go's sprintf + // code will understand. + argLen := len(args) + fmtArgs := make([]interface{}, argLen-1) + // Here we convert and assign. + for i, v := range args[1:] { + fmtArgs[i] = v.ToInterface() + } + // Call the helper + out := fmt.Sprintf(fs, fmtArgs...) + i.logger.Debug().Str("fn", "printf").Msg(out) + return &object.Void{} +} + +func (i *Interpreter) fnTriggerWebhook(args []object.Object) object.Object { + if len(args) != 1 { + return &object.Void{} + } + name := args[0].Inspect() + operations := *i.operations + operations = append(operations, *NewOperation(OpTriggerWebhook, name)) + i.operations = &operations + return &object.Void{} +} + +func (i *Interpreter) fnSendNotification(args []object.Object) object.Object { + operations := *i.operations + operations = append(operations, *NewOperation(OpSendNotification)) + i.operations = &operations + return &object.Void{} +} + +func (i *Interpreter) fnSetTitle(args []object.Object) object.Object { + if len(args) != 1 { + return &object.Void{} + } + value := args[0].Inspect() + operations := *i.operations + operations = append(operations, *NewOperation(OpSetTitle, value)) + i.operations = &operations + return &object.Void{} +} + +func (i *Interpreter) fnSetCategory(args []object.Object) object.Object { + if len(args) != 1 { + return &object.Void{} + } + value := args[0].Inspect() + operations := *i.operations + operations = append(operations, *NewOperation(OpSetCategory, value)) + i.operations = &operations + return &object.Void{} +} diff --git a/pkg/scripting/interpreter.go b/pkg/scripting/interpreter.go new file mode 100644 index 000000000..710353a11 --- /dev/null +++ b/pkg/scripting/interpreter.go @@ -0,0 +1,65 @@ +package scripting + +import ( + "context" + "fmt" + "sync" + + "github.com/rs/zerolog" + "github.com/skx/evalfilter/v2" +) + +// Interpreter is a script interpreter +type Interpreter struct { + eval *evalfilter.Eval + operations *OperationStack + mu sync.Mutex + logger zerolog.Logger +} + +// NewInterpreter create new script interpreter +func NewInterpreter(script string, logger zerolog.Logger) (*Interpreter, error) { + eval := evalfilter.New(script) + if err := eval.Prepare(); err != nil { + return nil, fmt.Errorf("unable to compile provided script: %w", err) + } + + operations := OperationStack{} + interpreter := &Interpreter{ + eval: eval, + operations: &operations, + logger: logger, + } + + // init the interpreter + interpreter.init() + + return interpreter, nil +} + +func (i *Interpreter) init() { + // deactivate unwanted functions + i.eval.AddFunction("getenv", fnNoOp) + // alter builtins functions + i.eval.AddFunction("print", i.fnPrint) + i.eval.AddFunction("printf", i.fnPrintf) + // add custom functions + i.eval.AddFunction("triggerWebhook", i.fnTriggerWebhook) + i.eval.AddFunction("sendNotification", i.fnSendNotification) + i.eval.AddFunction("setTitle", i.fnSetTitle) + i.eval.AddFunction("setCategory", i.fnSetCategory) +} + +// Exec a script by the interpreter +func (i *Interpreter) Exec(ctx context.Context, input ScriptInput) (OperationStack, error) { + i.mu.Lock() + defer i.mu.Unlock() + i.operations = &OperationStack{} + if result, err := i.eval.Run(input); err != nil { + return nil, fmt.Errorf("unable to execute script: %w", err) + } else if !result { + operations := append(*i.operations, *NewOperation(OpDrop)) + i.operations = &operations + } + return *i.operations, nil +} diff --git a/pkg/scripting/operation.go b/pkg/scripting/operation.go new file mode 100644 index 000000000..d38d4d884 --- /dev/null +++ b/pkg/scripting/operation.go @@ -0,0 +1,34 @@ +package scripting + +// OperationName is the operation name +type OperationName uint + +const ( + // OpDrop to drop an article + OpDrop OperationName = iota + // OpTriggerWebhook to trigger an outgoing webhook + OpTriggerWebhook + // OpSendNotification to send a notification to all user devices + OpSendNotification + // OpSetTitle to set article title + OpSetTitle + // OpSetCategory to set article category + OpSetCategory +) + +// Operation object +type Operation struct { + Name OperationName + Args []string +} + +// OperationStack is a stack of operation +type OperationStack []Operation + +// NewOperation create new operation +func NewOperation(name OperationName, args ...string) *Operation { + return &Operation{ + Name: name, + Args: args, + } +} diff --git a/pkg/scripting/test/engine_test.go b/pkg/scripting/test/engine_test.go new file mode 100644 index 000000000..50f116edb --- /dev/null +++ b/pkg/scripting/test/engine_test.go @@ -0,0 +1,35 @@ +package test + +import ( + "context" + "testing" + + "github.com/ncarlier/readflow/pkg/scripting" + "github.com/stretchr/testify/assert" +) + +func TestEngineSimpleScript(t *testing.T) { + script := ` +if (Title == "foo") { + printf("sending notification because Title = %s", Title); + sendNotification(); + return false; +} +return true; +` + engine := scripting.NewScriptEngine(10) + assert.NotNil(t, engine) + + input := scripting.ScriptInput{ + Title: "foo", + } + + operations, err := engine.Exec(context.Background(), script, input) + assert.Nil(t, err) + assert.Len(t, operations, 2) + + input.Title = "bar" + operations, err = engine.Exec(context.Background(), script, input) + assert.Nil(t, err) + assert.Len(t, operations, 0) +} diff --git a/pkg/scripting/test/interpreter_test.go b/pkg/scripting/test/interpreter_test.go new file mode 100644 index 000000000..e463c7ee0 --- /dev/null +++ b/pkg/scripting/test/interpreter_test.go @@ -0,0 +1,77 @@ +package test + +import ( + "context" + "strconv" + "sync" + "testing" + + "github.com/ncarlier/readflow/pkg/scripting" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" +) + +func TestInterpreterSimpleScript(t *testing.T) { + script := ` +if ("bar" in Tags) { + print("sending notification because Tags contains bar"); + sendNotification(); + return false; +} +return true; +` + + interpreter, err := scripting.NewInterpreter(script, log.Logger) + assert.Nil(t, err) + assert.NotNil(t, interpreter) + input := scripting.ScriptInput{ + Tags: []string{"foo", "bar"}, + } + operations, err := interpreter.Exec(context.Background(), input) + assert.Nil(t, err) + assert.Len(t, operations, 2) + operation := operations[0] + assert.Equal(t, scripting.OpSendNotification, operation.Name, "invalid operation") + operation = operations[1] + assert.Equal(t, scripting.OpDrop, operation.Name, "invalid operation") + + input.Tags = []string{"foo"} + operations, err = interpreter.Exec(context.Background(), input) + assert.Nil(t, err) + assert.Len(t, operations, 0) +} + +func TestInterpreterRaceCondition(t *testing.T) { + script := ` +printf("title=%s", Title); +index = int(Title) +if (index %2 == 0) { + return true; +} +return false; +` + + interpreter, err := scripting.NewInterpreter(script, log.Logger) + assert.Nil(t, err) + assert.NotNil(t, interpreter) + + count := 10 + wg := sync.WaitGroup{} + wg.Add(count) + results := make([]scripting.OperationStack, count) + + for i := 0; i < count; i++ { + go func(index int) { + input := scripting.ScriptInput{Title: strconv.Itoa(index)} + ops, _ := interpreter.Exec(context.Background(), input) + results[index] = ops + wg.Done() + }(i) + } + + wg.Wait() + for i := 0; i < count; i++ { + result := results[i] + assert.Len(t, result, i%2, "invalid test case #%d", i) + } +} diff --git a/pkg/scripting/types.go b/pkg/scripting/types.go new file mode 100644 index 000000000..68ba0f5c7 --- /dev/null +++ b/pkg/scripting/types.go @@ -0,0 +1,10 @@ +package scripting + +// ScriptInput is the paylod passed to the script +type ScriptInput struct { + URL string + HTML string + Text string + Title string + Tags []string +} diff --git a/pkg/service/articles_create.go b/pkg/service/articles_create.go index 216ea1123..998e6e4e1 100644 --- a/pkg/service/articles_create.go +++ b/pkg/service/articles_create.go @@ -2,9 +2,12 @@ package service import ( "context" + "fmt" + "github.com/ncarlier/readflow/pkg/constant" "github.com/ncarlier/readflow/pkg/event" "github.com/ncarlier/readflow/pkg/model" + "github.com/ncarlier/readflow/pkg/scripting" // activate all content providers _ "github.com/ncarlier/readflow/pkg/scraper/content-provider/all" @@ -46,21 +49,9 @@ func (reg *Registry) CreateArticle(ctx context.Context, form model.ArticleCreate } // TODO validate article! - - // Get category if specified - var category *model.Category + // validate category if form.CategoryID != nil { - cat, err := reg.GetCategory(ctx, *form.CategoryID) - if err != nil { - logger.Err(err).Msg(unableToCreateArticleErrorMsg) - return nil, err - } - category = cat - } - - if category == nil { - // Process article by the rule engine - if err := reg.ProcessArticleByRuleEngine(ctx, &form); err != nil { + if _, err := reg.GetCategory(ctx, *form.CategoryID); err != nil { logger.Err(err).Msg(unableToCreateArticleErrorMsg) return nil, err } @@ -85,6 +76,25 @@ func (reg *Registry) CreateArticle(ctx context.Context, form model.ArticleCreate form.HTML = &content } + var ops scripting.OperationStack + if alias := ctx.Value(constant.ContextIncomingWebhookAlias); alias != nil { + // Process article by the script engine if comming from webhook + if ops, err = reg.ProcessArticleByScriptEngine(ctx, alias.(string), &form); err != nil { + debug.Err(err).Msg("unable to process article by script engine") + text := err.Error() + if form.Text != nil { + text = fmt.Sprintf("%s\n%s", text, *form.Text) + } + form.Text = &text + } + } + // Drop if asked + for _, v := range ops { + if v.Name == scripting.OpDrop { + return nil, nil + } + } + debug.Msg("creating article...") article, err := reg.db.CreateArticleForUser(uid, form) if err != nil { @@ -92,6 +102,7 @@ func (reg *Registry) CreateArticle(ctx context.Context, form model.ArticleCreate return nil, err } logger.Uint("id", article.ID).Msg("article created") + // TODO trigger async actions (notification/webhook) event.Emit(event.CreateArticle, *article) return article, nil } diff --git a/pkg/service/categories.go b/pkg/service/categories.go index 74cb96d1b..03caa1796 100644 --- a/pkg/service/categories.go +++ b/pkg/service/categories.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/ncarlier/readflow/pkg/model" - ruleengine "github.com/ncarlier/readflow/pkg/rule-engine" ) // GetCategories get categories from current user @@ -70,14 +69,6 @@ func (reg *Registry) CreateCategory(ctx context.Context, form model.CategoryCrea } } - // Validate category's rule - if err := validateCategoryRule(form.Rule); err != nil { - reg.logger.Info().Err(err).Uint( - "uid", uid, - ).Msg("invalid category rule") - return nil, err - } - // Create category result, err := reg.db.CreateCategoryForUser(uid, form) if err != nil { @@ -87,9 +78,6 @@ func (reg *Registry) CreateCategory(ctx context.Context, form model.CategoryCrea return nil, err } - // Force to refresh the rule engine cache - reg.ruleEngineCache.Evict(uid) - return result, err } @@ -97,14 +85,6 @@ func (reg *Registry) CreateCategory(ctx context.Context, form model.CategoryCrea func (reg *Registry) UpdateCategory(ctx context.Context, form model.CategoryUpdateForm) (*model.Category, error) { uid := getCurrentUserIDFromContext(ctx) - // Validate category's rule - if err := validateCategoryRule(form.Rule); err != nil { - reg.logger.Info().Err(err).Uint( - "uid", uid, - ).Msg("invalid category rule") - return nil, err - } - // Update category result, err := reg.db.UpdateCategoryForUser(uid, form) if err != nil { @@ -116,9 +96,6 @@ func (reg *Registry) UpdateCategory(ctx context.Context, form model.CategoryUpda return nil, err } - // Force to refresh the rule engine cache - reg.ruleEngineCache.Evict(uid) - return result, err } @@ -139,9 +116,6 @@ func (reg *Registry) DeleteCategory(ctx context.Context, id uint) (*model.Catego return nil, err } - // Force to refresh the rule engine cache - reg.ruleEngineCache.Evict(uid) - return category, nil } @@ -161,21 +135,5 @@ func (reg *Registry) DeleteCategories(ctx context.Context, ids []uint) (int64, e "uid", uid, ).Str("ids", idsStr).Int64("nb", nb).Msg("categories deleted") - // Force to refresh the rule engine cache - reg.ruleEngineCache.Evict(uid) - return nb, nil } - -func validateCategoryRule(rule *string) error { - if rule == nil { - return nil - } - // Create dummy category in order to validate rule - category := model.Category{ - Rule: rule, - } - // Validate category's rule - _, err := ruleengine.NewRuleProcessor(category) - return err -} diff --git a/pkg/service/registry.go b/pkg/service/registry.go index 6387e1c9d..7d9a2c75d 100644 --- a/pkg/service/registry.go +++ b/pkg/service/registry.go @@ -10,9 +10,9 @@ import ( "github.com/ncarlier/readflow/pkg/helper" "github.com/ncarlier/readflow/pkg/model" ratelimiter "github.com/ncarlier/readflow/pkg/rate-limiter" - ruleengine "github.com/ncarlier/readflow/pkg/rule-engine" "github.com/ncarlier/readflow/pkg/sanitizer" "github.com/ncarlier/readflow/pkg/scraper" + "github.com/ncarlier/readflow/pkg/scripting" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -24,13 +24,13 @@ type Registry struct { conf config.Config db db.DB logger zerolog.Logger - ruleEngineCache *ruleengine.Cache downloadCache cache.Cache properties *model.Properties webScraper scraper.WebScraper downloader exporter.Downloader hashid *helper.HashIDHandler notificationRateLimiter ratelimiter.RateLimiter + scriptEngine *scripting.ScriptEngine sanitizer *sanitizer.Sanitizer } @@ -57,13 +57,13 @@ func Configure(conf config.Config, database db.DB, downloadCache cache.Cache) er conf: conf, db: database, logger: log.With().Str("component", "service").Logger(), - ruleEngineCache: ruleengine.NewRuleEngineCache(1024), downloadCache: downloadCache, webScraper: webScraper, downloader: exporter.NewInternalDownloader(downloadCache, 10, constant.DefaultTimeout), hashid: hashid, notificationRateLimiter: notificationRateLimiter, sanitizer: sanitizer.NewSanitizer(blockList), + scriptEngine: scripting.NewScriptEngine(128), } return instance.initProperties() } diff --git a/pkg/service/rules.go b/pkg/service/rules.go deleted file mode 100644 index a599d4ee6..000000000 --- a/pkg/service/rules.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "context" - - "github.com/ncarlier/readflow/pkg/model" - ruleengine "github.com/ncarlier/readflow/pkg/rule-engine" -) - -// ProcessArticleByRuleEngine apply user's rules on the article -func (reg *Registry) ProcessArticleByRuleEngine(ctx context.Context, article *model.ArticleCreateForm) error { - uid := getCurrentUserIDFromContext(ctx) - // Retrieve pipeline from cache - pipeline := reg.ruleEngineCache.Get(uid) - if pipeline == nil { - reg.logger.Debug().Uint( - "uid", uid, - ).Msg("loading rules into the cache") - // Init pipeline if not in cache - categories, err := reg.GetCategories(ctx) - if err != nil { - return err - } - pipeline, err = ruleengine.NewProcessorsPipeline(categories) - if err != nil { - return err - } - reg.ruleEngineCache.Set(uid, pipeline) - } - applied, err := pipeline.Apply(ctx, article) - if err != nil { - reg.logger.Info().Err(err).Uint( - "uid", uid, - ).Str("title", article.Title).Msg("unable to apply rules on the article") - return err - } - if applied { - reg.logger.Debug().Uint( - "uid", uid, - ).Str("title", article.Title).Uint( - "category", *article.CategoryID, - ).Msg("rule applied on the article") - } - return nil -} diff --git a/pkg/service/scripting.go b/pkg/service/scripting.go new file mode 100644 index 000000000..ff8826e94 --- /dev/null +++ b/pkg/service/scripting.go @@ -0,0 +1,70 @@ +package service + +import ( + "context" + "strings" + "time" + + "github.com/ncarlier/readflow/pkg/model" + "github.com/ncarlier/readflow/pkg/scripting" +) + +func mapArticleCreateFromToScriptInput(article *model.ArticleCreateForm) *scripting.ScriptInput { + input := scripting.ScriptInput{ + Title: article.Title, + } + if article.URL != nil { + input.URL = *article.URL + } + if article.Text != nil { + input.Text = *article.Text + } + if article.HTML != nil { + input.HTML = *article.HTML + } + if article.Tags != nil { + input.Tags = strings.Split(*article.Tags, ",") + } + return &input +} + +// ProcessArticleByScriptEngine apply user's script on the article +func (reg *Registry) ProcessArticleByScriptEngine(ctx context.Context, alias string, article *model.ArticleCreateForm) (scripting.OperationStack, error) { + uid := getCurrentUserIDFromContext(ctx) + + noops := scripting.OperationStack{} + + // retrieve webhook + webhook, err := reg.db.GetIncomingWebhookByUserAndAlias(uid, alias) + if err != nil || webhook == nil { + return noops, err + } + + // limit execution time to 1 sec + ctx, cancel := context.WithTimeout(ctx, time.Duration(time.Second)) + defer cancel() + + // build input script object + input := mapArticleCreateFromToScriptInput(article) + + // exec user's script + ops, err := reg.scriptEngine.Exec(ctx, webhook.Script, *input) + if err != nil { + return noops, err + } + + // Apply setter + for _, op := range ops { + switch op.Name { + case scripting.OpSetCategory: + // Set category + if cat, err := reg.db.GetCategoryByUserAndTitle(uid, op.Args[0]); err == nil { + article.CategoryID = cat.ID + } + case scripting.OpSetTitle: + // Set title + article.Title = op.Args[0] + } + } + return ops, err +} diff --git a/pkg/service/test/articles_create_test.go b/pkg/service/test/articles_create_test.go index 70b686d21..e9b721d81 100644 --- a/pkg/service/test/articles_create_test.go +++ b/pkg/service/test/articles_create_test.go @@ -55,25 +55,6 @@ func TestCreateArticleInCategory(t *testing.T) { assert.Equal(t, *cat.ID, *art.CategoryID) } -func TestCreateArticleWithRuleEngine(t *testing.T) { - teardownTestCase := setupTestCase(t) - defer teardownTestCase(t) - - // Create category with rule - uid := *testUser.ID - formBuilder := model.NewCategoryCreateFormBuilder() - rule := "title matches \"^Test\"" - form := formBuilder.Random().Rule(&rule).Build() - cat, err := service.Lookup().CreateCategory(testContext, *form) - assert.Nil(t, err) - assert.Equal(t, form.Title, cat.Title) - assert.Equal(t, uid, *cat.UserID) - - // Create article - art := assertNewArticle(t, "TestCreateArticleWithRuleEngine") - assert.Equal(t, *cat.ID, *art.CategoryID) -} - func TestCreateArticlesExceedingQuota(t *testing.T) { teardownTestCase := setupTestCase(t) defer teardownTestCase(t) diff --git a/pkg/service/test/articles_scripting_test.go b/pkg/service/test/articles_scripting_test.go new file mode 100644 index 000000000..6c56d5c24 --- /dev/null +++ b/pkg/service/test/articles_scripting_test.go @@ -0,0 +1,43 @@ +package test + +import ( + "context" + "fmt" + "testing" + + "github.com/ncarlier/readflow/pkg/constant" + "github.com/ncarlier/readflow/pkg/model" + "github.com/ncarlier/readflow/pkg/service" + "github.com/stretchr/testify/assert" +) + +func TestCreateArticleWithScriptEngine(t *testing.T) { + teardownTestCase := setupTestCase(t) + defer teardownTestCase(t) + + // Create category + cat := assertNewCategory(t) + + // Create incoming webhook + script := fmt.Sprintf(` +if ( Title ~= /script/i ) { + setCategory("%s"); + return true; +} +return false; +`, cat.Title) + assertNewIncomingWebhook(t, "foo", script) + ctx := context.WithValue(testContext, constant.ContextIncomingWebhookAlias, "foo") + + // Create article + form := model.ArticleCreateForm{ + Title: "TestCreateArticleWithScriptEngine", + } + opts := service.ArticleCreationOptions{} + art, err := service.Lookup().CreateArticle(ctx, form, opts) + assert.Nil(t, err) + assert.Equal(t, form.Title, art.Title) + assert.Equal(t, *testUser.ID, art.UserID) + assert.NotNil(t, art.CategoryID) + assert.Equal(t, *cat.ID, *art.CategoryID) +} diff --git a/pkg/service/test/helper.go b/pkg/service/test/helper.go index 6f2d33f54..040f8a7c6 100644 --- a/pkg/service/test/helper.go +++ b/pkg/service/test/helper.go @@ -30,3 +30,15 @@ func assertNewCategory(t *testing.T) *model.Category { assert.Equal(t, *testUser.ID, *cat.UserID) return cat } + +func assertNewIncomingWebhook(t *testing.T, alias string, script string) *model.IncomingWebhook { + builder := model.NewIncomingWebhookCreateFormBuilder() + form := builder.Alias(alias).Script(script).Build() + webhook, err := service.Lookup().CreateIncomingWebhook(testContext, *form) + assert.Nil(t, err) + assert.Equal(t, *testUser.ID, webhook.UserID) + assert.Equal(t, alias, webhook.Alias) + assert.Equal(t, script, webhook.Script) + assert.NotEmpty(t, webhook.Token) + return webhook +} diff --git a/scripts/payload.json b/scripts/payload.json index 90a1c45f6..21f7875bb 100644 --- a/scripts/payload.json +++ b/scripts/payload.json @@ -1,10 +1,13 @@ [ { - "title": "test 001" + "title": "test 001", + "tags": "foo,bar" }, { "title": "test 002", - "html": "

test 002

" + "html": "

test 002

", + "tags": "foo,bar" }, { - "url": "https://en.wikipedia.org/wiki/Special:Random" + "url": "https://en.wikipedia.org/wiki/Special:Random", + "tags": "bar" } ] diff --git a/ui/src/articles/components/EditArticleForm.tsx b/ui/src/articles/components/EditArticleForm.tsx index f58ab2e2b..03b55809f 100644 --- a/ui/src/articles/components/EditArticleForm.tsx +++ b/ui/src/articles/components/EditArticleForm.tsx @@ -76,8 +76,8 @@ export const EditArticleForm = ({ article, onSuccess, onCancel }: Props) => {
{errorMessage != null && {errorMessage}}
- - + + diff --git a/ui/src/categories/models.ts b/ui/src/categories/models.ts index bd3645503..0bd7f5297 100644 --- a/ui/src/categories/models.ts +++ b/ui/src/categories/models.ts @@ -1,8 +1,6 @@ export interface Category { id?: number title: string - rule: string | null - notification_strategy: 'none' | 'individual' | 'global' inbox?: number created_at?: string updated_at?: string diff --git a/ui/src/categories/queries.ts b/ui/src/categories/queries.ts index a888bfafd..e0517b2bd 100644 --- a/ui/src/categories/queries.ts +++ b/ui/src/categories/queries.ts @@ -9,8 +9,6 @@ export const GetCategories = gql` entries { id title - rule - notification_strategy inbox created_at updated_at @@ -24,8 +22,6 @@ export const GetCategory = gql` category(id: $id) { id title - rule - notification_strategy created_at updated_at } @@ -42,14 +38,10 @@ export const CreateOrUpdateCategory = gql` mutation createOrUpdateCategory( $id: ID $title: String - $rule: String - $notification_strategy: notificationStrategy ) { - createOrUpdateCategory(id: $id, title: $title, rule: $rule, notification_strategy: $notification_strategy) { + createOrUpdateCategory(id: $id, title: $title) { id title - rule - notification_strategy created_at updated_at } diff --git a/ui/src/components/FormInputField.tsx b/ui/src/components/FormInputField.tsx index 632d941ea..430066a16 100644 --- a/ui/src/components/FormInputField.tsx +++ b/ui/src/components/FormInputField.tsx @@ -7,7 +7,7 @@ interface Props { label: string required?: boolean pattern?: string - maxlength?: string + maxLength?: number readOnly?: boolean autoFocus?: boolean error?: string diff --git a/ui/src/components/FormTextareaField.tsx b/ui/src/components/FormTextareaField.tsx index 49dc813d2..8fdaa19d4 100644 --- a/ui/src/components/FormTextareaField.tsx +++ b/ui/src/components/FormTextareaField.tsx @@ -7,7 +7,8 @@ interface Props { label: string required?: boolean readOnly?: boolean - maxlength?: string + pattern?: string + maxLength?: number error?: string children?: ReactNode } diff --git a/ui/src/settings/categories/AddCategoryForm.tsx b/ui/src/settings/categories/AddCategoryForm.tsx index 93968adfc..c980254db 100644 --- a/ui/src/settings/categories/AddCategoryForm.tsx +++ b/ui/src/settings/categories/AddCategoryForm.tsx @@ -12,9 +12,6 @@ import { Button, ErrorPanel, FormInputField, - FormSelectField, - FormTextareaField, - HelpLink, Panel, } from '../../components' import { getGQLError, isValidForm } from '../../helpers' @@ -22,18 +19,14 @@ import { usePageTitle } from '../../hooks' interface AddCategoryFormFields { title: string - rule: string - notification_strategy: 'none' | 'global' | 'individual' } const AddCategoryForm = ({ history }: RouteComponentProps) => { usePageTitle('Settings - Add new category') const [errorMessage, setErrorMessage] = useState(null) - const [formState, { text, textarea, select }] = useFormState({ + const [formState, { text }] = useFormState({ title: '', - rule: '', - notification_strategy: 'none', }) const [addCategoryMutation] = useMutation(CreateOrUpdateCategory) const { showMessage } = useMessage() @@ -75,20 +68,7 @@ const AddCategoryForm = ({ history }: RouteComponentProps) => {
{errorMessage != null && {errorMessage}} - - - View rule syntax - - - - - - +
diff --git a/ui/src/settings/categories/CategoriesTab.module.css b/ui/src/settings/categories/CategoriesTab.module.css deleted file mode 100644 index 0d1777584..000000000 --- a/ui/src/settings/categories/CategoriesTab.module.css +++ /dev/null @@ -1,7 +0,0 @@ -.rule { - text-overflow: ellipsis; - overflow: hidden; - max-width: 10em; - display: inline-block; - white-space: nowrap; -} diff --git a/ui/src/settings/categories/CategoriesTab.tsx b/ui/src/settings/categories/CategoriesTab.tsx index db44a21af..b46e40deb 100644 --- a/ui/src/settings/categories/CategoriesTab.tsx +++ b/ui/src/settings/categories/CategoriesTab.tsx @@ -4,7 +4,6 @@ import { useModal } from 'react-modal-hook' import { RouteComponentProps } from 'react-router' import { Link } from 'react-router-dom' -import classes from './CategoriesTab.module.css' import { updateCacheAfterDelete } from '../../categories/cache' import { Category, @@ -40,12 +39,6 @@ const CategoryDates = ({ val }: { val: Category }) => ( ) -const CategoryRule = ({ val: { rule } }: { val: Category }) => ( - - {rule || '-'} - -) - const definition = [ { title: 'Title', @@ -55,14 +48,6 @@ const definition = [ ), }, - { - title: 'Rule', - render: (val: Category) => , - }, - { - title: 'Notification strategy', - render: (val: Category) => val.notification_strategy, - }, { title: 'Date(s)', render: (val: Category) => , diff --git a/ui/src/settings/categories/EditCategoryForm.tsx b/ui/src/settings/categories/EditCategoryForm.tsx index 287d0b17f..7c630cf72 100644 --- a/ui/src/settings/categories/EditCategoryForm.tsx +++ b/ui/src/settings/categories/EditCategoryForm.tsx @@ -7,13 +7,11 @@ import { useFormState } from 'react-use-form-state' import { Category, CreateOrUpdateCategoryResponse } from '../../categories/models' import { CreateOrUpdateCategory } from '../../categories/queries' import { useMessage } from '../../contexts' -import { Button, ErrorPanel, FormInputField, FormSelectField, FormTextareaField, HelpLink } from '../../components' +import { Button, ErrorPanel, FormInputField } from '../../components' import { getGQLError, isValidForm } from '../../helpers' interface EditCategoryFormFields { title: string - rule: string - notification_strategy: 'none' | 'global' | 'individual' } interface Props { @@ -23,10 +21,8 @@ interface Props { const EditCategoryForm = ({ category, history }: Props) => { const [errorMessage, setErrorMessage] = useState(null) - const [formState, { text, textarea, select }] = useFormState({ + const [formState, { text }] = useFormState({ title: category.title, - rule: category.rule ? category.rule : '', - notification_strategy: category.notification_strategy, }) const [editCategoryMutation] = useMutation(CreateOrUpdateCategory) const { showMessage } = useMessage() @@ -67,20 +63,7 @@ const EditCategoryForm = ({ category, history }: Props) => {
{errorMessage != null && {errorMessage}}
- - - View rule syntax - - - - - - +
diff --git a/ui/src/settings/intergrations/incoming-webhook/AddIncomingWebhookForm.tsx b/ui/src/settings/intergrations/incoming-webhook/AddIncomingWebhookForm.tsx index 143897502..97d6957b2 100644 --- a/ui/src/settings/intergrations/incoming-webhook/AddIncomingWebhookForm.tsx +++ b/ui/src/settings/intergrations/incoming-webhook/AddIncomingWebhookForm.tsx @@ -5,7 +5,7 @@ import { Link } from 'react-router-dom' import { useFormState } from 'react-use-form-state' import { useMessage } from '../../../contexts' -import { Button, ErrorPanel, FormInputField, Panel } from '../../../components' +import { Button, ErrorPanel, FormInputField, FormTextareaField, HelpLink, Panel } from '../../../components' import { getGQLError, isValidForm } from '../../../helpers' import { usePageTitle } from '../../../hooks' import { updateCacheAfterCreate } from './cache' @@ -15,6 +15,7 @@ import { CreateOrUpdateIncomingWebhook } from './queries' interface AddIncomingWebhookFormFields { alias: string + script: string } type AllProps = RouteComponentProps @@ -23,7 +24,9 @@ export default ({ history }: AllProps) => { usePageTitle('Settings - Add new incoming webhook') const [errorMessage, setErrorMessage] = useState(null) - const [formState, { text }] = useFormState() + const [formState, { text, textarea }] = useFormState({ + script: 'return true;' + }) const [addIncomingWebhookMutation] = useMutation< CreateOrUpdateIncomingWebhookResponse, CreateOrUpdateIncomingWebhookRequest @@ -56,8 +59,7 @@ export default ({ history }: AllProps) => { setErrorMessage('Please fill out correctly the mandatory fields.') return } - const { alias } = formState.values - addIncomingWebhook({ alias }) + addIncomingWebhook(formState.values) }, [formState, addIncomingWebhook] ) @@ -71,7 +73,10 @@ export default ({ history }: AllProps) => { {errorMessage != null && {errorMessage}}
- + + + View script syntax +