Skip to content

Commit

Permalink
chore: implement memo property runner
Browse files Browse the repository at this point in the history
  • Loading branch information
boojack committed Aug 20, 2024
1 parent f4d6675 commit d1280bc
Show file tree
Hide file tree
Showing 6 changed files with 214 additions and 129 deletions.
63 changes: 7 additions & 56 deletions server/router/api/v1/memo_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"context"
"fmt"
"log/slog"
"slices"
"time"
"unicode/utf8"

Expand All @@ -28,6 +27,7 @@ import (
"github.com/usememos/memos/plugin/webhook"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store"
memoproperty "github.com/usememos/memos/server/runner/memo_property"
"github.com/usememos/memos/store"
)

Expand Down Expand Up @@ -60,8 +60,9 @@ func (s *APIV1Service) CreateMemo(ctx context.Context, request *v1pb.CreateMemoR
}
if len(create.Content) > contentLengthLimit {

Check failure on line 61 in server/router/api/v1/memo_service.go

View workflow job for this annotation

GitHub Actions / go-static-checks

empty-lines: extra empty line at the end of a block (revive)
return nil, status.Errorf(codes.InvalidArgument, "content too long (max %d characters)", contentLengthLimit)

}
property, err := getMemoPropertyFromContent(create.Content)
property, err := memoproperty.GetMemoPropertyFromContent(create.Content)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err)
}
Expand Down Expand Up @@ -247,7 +248,7 @@ func (s *APIV1Service) UpdateMemo(ctx context.Context, request *v1pb.UpdateMemoR
}
update.Content = &request.Memo.Content

property, err := getMemoPropertyFromContent(*update.Content)
property, err := memoproperty.GetMemoPropertyFromContent(*update.Content)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err)
}
Expand Down Expand Up @@ -610,7 +611,7 @@ func (s *APIV1Service) RebuildMemoProperty(ctx context.Context, request *v1pb.Re
}

for _, memo := range memos {
property, err := getMemoPropertyFromContent(memo.Content)
property, err := memoproperty.GetMemoPropertyFromContent(memo.Content)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err)
}
Expand Down Expand Up @@ -691,14 +692,14 @@ func (s *APIV1Service) RenameMemoTag(ctx context.Context, request *v1pb.RenameMe
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to parse memo: %v", err)
}
TraverseASTNodes(nodes, func(node ast.Node) {
memoproperty.TraverseASTNodes(nodes, func(node ast.Node) {
if tag, ok := node.(*ast.Tag); ok && tag.Content == request.OldTag {
tag.Content = request.NewTag
}
})
content := restore.Restore(nodes)

property, err := getMemoPropertyFromContent(content)
property, err := memoproperty.GetMemoPropertyFromContent(content)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo property: %v", err)
}
Expand Down Expand Up @@ -1127,56 +1128,6 @@ func findMemoField(callExpr *expr.Expr_Call, filter *MemoFilter) {
}
}

func getMemoPropertyFromContent(content string) (*storepb.MemoPayload_Property, error) {
nodes, err := parser.Parse(tokenizer.Tokenize(content))
if err != nil {
return nil, errors.Wrap(err, "failed to parse content")
}

property := &storepb.MemoPayload_Property{}
TraverseASTNodes(nodes, func(node ast.Node) {
switch n := node.(type) {
case *ast.Tag:
tag := n.Content
if !slices.Contains(property.Tags, tag) {
property.Tags = append(property.Tags, tag)
}
case *ast.Link, *ast.AutoLink:
property.HasLink = true
case *ast.TaskList:
property.HasTaskList = true
if !n.Complete {
property.HasIncompleteTasks = true
}
case *ast.Code, *ast.CodeBlock:
property.HasCode = true
}
})
return property, nil
}

func TraverseASTNodes(nodes []ast.Node, fn func(ast.Node)) {
for _, node := range nodes {
fn(node)
switch n := node.(type) {
case *ast.Paragraph:
TraverseASTNodes(n.Children, fn)
case *ast.Heading:
TraverseASTNodes(n.Children, fn)
case *ast.Blockquote:
TraverseASTNodes(n.Children, fn)
case *ast.OrderedList:
TraverseASTNodes(n.Children, fn)
case *ast.UnorderedList:
TraverseASTNodes(n.Children, fn)
case *ast.TaskList:
TraverseASTNodes(n.Children, fn)
case *ast.Bold:
TraverseASTNodes(n.Children, fn)
}
}
}

// DispatchMemoCreatedWebhook dispatches webhook when memo is created.
func (s *APIV1Service) DispatchMemoCreatedWebhook(ctx context.Context, memo *v1pb.Memo) error {
return s.dispatchMemoRelatedWebhook(ctx, memo, "memos.memo.created")
Expand Down
120 changes: 120 additions & 0 deletions server/runner/memo_property/runner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package memoproperty

import (
"context"
"log/slog"
"slices"
"time"

"github.com/pkg/errors"
"github.com/usememos/gomark/ast"
"github.com/usememos/gomark/parser"
"github.com/usememos/gomark/parser/tokenizer"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)

type Runner struct {
Store *store.Store
}

func NewRunner(store *store.Store) *Runner {
return &Runner{
Store: store,
}
}

// Schedule runner every 12 hours.
const runnerInterval = time.Hour * 12

func (r *Runner) Run(ctx context.Context) {
ticker := time.NewTicker(runnerInterval)
defer ticker.Stop()

for {
select {
case <-ticker.C:
r.RunOnce(ctx)
case <-ctx.Done():
return
}
}
}

func (r *Runner) RunOnce(ctx context.Context) {
emptyPayload := "{}"
memos, err := r.Store.ListMemos(ctx, &store.FindMemo{
PayloadFind: &store.FindMemoPayload{
Raw: &emptyPayload,
},
})
if err != nil {
slog.Error("failed to list memos", "err", err)
return
}

for _, memo := range memos {
property, err := GetMemoPropertyFromContent(memo.Content)
if err != nil {
slog.Error("failed to get memo property", "err", err)
continue
}
memo.Payload.Property = property
if err := r.Store.UpdateMemo(ctx, &store.UpdateMemo{
ID: memo.ID,
Payload: memo.Payload,
}); err != nil {
slog.Error("failed to update memo", "err", err)
}
}
}

func GetMemoPropertyFromContent(content string) (*storepb.MemoPayload_Property, error) {
nodes, err := parser.Parse(tokenizer.Tokenize(content))
if err != nil {
return nil, errors.Wrap(err, "failed to parse content")
}

property := &storepb.MemoPayload_Property{}
TraverseASTNodes(nodes, func(node ast.Node) {
switch n := node.(type) {
case *ast.Tag:
tag := n.Content
if !slices.Contains(property.Tags, tag) {
property.Tags = append(property.Tags, tag)
}
case *ast.Link, *ast.AutoLink:
property.HasLink = true
case *ast.TaskList:
property.HasTaskList = true
if !n.Complete {
property.HasIncompleteTasks = true
}
case *ast.Code, *ast.CodeBlock:
property.HasCode = true
}
})
return property, nil
}

func TraverseASTNodes(nodes []ast.Node, fn func(ast.Node)) {
for _, node := range nodes {
fn(node)
switch n := node.(type) {
case *ast.Paragraph:
TraverseASTNodes(n.Children, fn)
case *ast.Heading:
TraverseASTNodes(n.Children, fn)
case *ast.Blockquote:
TraverseASTNodes(n.Children, fn)
case *ast.OrderedList:
TraverseASTNodes(n.Children, fn)
case *ast.UnorderedList:
TraverseASTNodes(n.Children, fn)
case *ast.TaskList:
TraverseASTNodes(n.Children, fn)
case *ast.Bold:
TraverseASTNodes(n.Children, fn)
}
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package s3objectpresigner
package s3presign

import (
"context"
Expand All @@ -12,25 +12,45 @@ import (
"github.com/usememos/memos/store"
)

// nolint
type S3ObjectPresigner struct {
type Runner struct {
Store *store.Store
}

func NewS3ObjectPresigner(store *store.Store) *S3ObjectPresigner {
return &S3ObjectPresigner{
func NewRunner(store *store.Store) *Runner {
return &Runner{
Store: store,
}
}

func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) {
workspaceStorageSetting, err := p.Store.GetWorkspaceStorageSetting(ctx)
// Schedule runner every 12 hours.
const runnerInterval = time.Hour * 12

func (r *Runner) Run(ctx context.Context) {
ticker := time.NewTicker(runnerInterval)
defer ticker.Stop()

for {
select {
case <-ticker.C:
r.RunOnce(ctx)
case <-ctx.Done():
return
}
}
}

func (r *Runner) RunOnce(ctx context.Context) {
r.CheckAndPresign(ctx)
}

func (r *Runner) CheckAndPresign(ctx context.Context) {
workspaceStorageSetting, err := r.Store.GetWorkspaceStorageSetting(ctx)
if err != nil {
return
}

s3StorageType := storepb.ResourceStorageType_S3
resources, err := p.Store.ListResources(ctx, &store.FindResource{
resources, err := r.Store.ListResources(ctx, &store.FindResource{
GetBlob: false,
StorageType: &s3StorageType,
})
Expand Down Expand Up @@ -73,7 +93,7 @@ func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) {
}
s3ObjectPayload.S3Config = s3Config
s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now())
if err := p.Store.UpdateResource(ctx, &store.UpdateResource{
if err := r.Store.UpdateResource(ctx, &store.UpdateResource{
ID: resource.ID,
Reference: &presignURL,
Payload: &storepb.ResourcePayload{
Expand All @@ -86,21 +106,3 @@ func (p *S3ObjectPresigner) CheckAndPresign(ctx context.Context) {
}
}
}

func (p *S3ObjectPresigner) Start(ctx context.Context) {
p.CheckAndPresign(ctx)

// Schedule runner every 24 hours.
ticker := time.NewTicker(24 * time.Hour)
defer ticker.Stop()

for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}

p.CheckAndPresign(ctx)
}
}
Loading

0 comments on commit d1280bc

Please sign in to comment.