Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

exp/lighthorizon: enforce the limit from request on the response size #4431

Merged
merged 18 commits into from
Jun 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c8042ff
#4430: wip code for further reveiw on intended change for index usage
sreuland Jun 13, 2022
f05d62a
#4430: enforce limit parameter from request on response size for txs
sreuland Jun 13, 2022
2103514
#4430: fixed static shadow errors
sreuland Jun 14, 2022
3108347
#4330: silenced some static check warnings
sreuland Jun 14, 2022
4204af6
Merge remote-tracking branch 'upstream/lighthorizon' into lighthorizon
sreuland Jun 14, 2022
99cf679
#4430: added a starter for api docs in oapi format
sreuland Jun 20, 2022
5783f26
#4430: added page limit processing to operations endpoint
sreuland Jun 21, 2022
70bda9c
#4430: removed some left over debug code
sreuland Jun 21, 2022
d5e26e5
Merge remote-tracking branch 'upstream/lighthorizon' into lighthorizon
sreuland Jun 23, 2022
a5a8b83
#4430: increment tx order num in all cases for each ledger read opera…
sreuland Jun 23, 2022
947070c
#4430: refactored paging params validation
sreuland Jun 23, 2022
eb918f5
#4430: used correct index constants
sreuland Jun 23, 2022
9e401e5
#4430: more verbose error messages for invalid tx id request param
sreuland Jun 23, 2022
a6d26c4
#4430: fixed transaction toid increment on read loop
sreuland Jun 24, 2022
c18d113
#4430: fixed go formatting warning
sreuland Jun 24, 2022
b11efd9
#4430: added unit test coverage, use adapter pattern for archive
sreuland Jun 25, 2022
7fc04ae
#4430: added some docs on ingest archive interface
sreuland Jun 29, 2022
7ed242f
#4430: fixed go formatting warning
sreuland Jun 29, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions exp/lighthorizon/actions/apidocs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package actions

import (
"net/http"
)

func ApiDocs() func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
r.URL.Scheme = "http"
r.URL.Host = "localhost:8080"

if r.Method != "GET" {
sendErrorResponse(w, http.StatusMethodNotAllowed, "")
return
}

p, err := staticFiles.ReadFile("static/api_docs.yml")
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/openapi+yaml")
w.Write(p)
}
}
94 changes: 94 additions & 0 deletions exp/lighthorizon/actions/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package actions

import (
"embed"
"encoding/json"
"net/http"
"net/url"
"strconv"

"github.com/stellar/go/support/log"
"github.com/stellar/go/support/render/hal"
)

var (
//go:embed static
staticFiles embed.FS
)

type Order string
type ErrorMessage string

const (
OrderAsc Order = "asc"
OrderDesc Order = "desc"
)

const (
ServerError ErrorMessage = "Error: A problem occurred on the server while processing request"
InvalidPagingParameters ErrorMessage = "Error: Invalid paging parameters"
)

type Pagination struct {
Limit int64
Cursor int64
Order
}

func sendPageResponse(w http.ResponseWriter, page hal.Page) {
encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
err := encoder.Encode(page)
if err != nil {
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
}
}

func sendErrorResponse(w http.ResponseWriter, errorCode int, errorMsg string) {
if errorMsg != "" {
http.Error(w, errorMsg, errorCode)
} else {
http.Error(w, string(ServerError), errorCode)
}
}

func RequestUnaryParam(r *http.Request, paramName string) (string, error) {
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return "", err
}
return query.Get(paramName), nil
}

func Paging(r *http.Request) (Pagination, error) {
paginate := Pagination{
Order: OrderAsc,
}

if cursorRequested, err := RequestUnaryParam(r, "cursor"); err != nil {
return Pagination{}, err
} else if cursorRequested != "" {
paginate.Cursor, err = strconv.ParseInt(cursorRequested, 10, 64)
if err != nil {
return Pagination{}, err
}
}

if limitRequested, err := RequestUnaryParam(r, "limit"); err != nil {
return Pagination{}, err
} else if limitRequested != "" {
paginate.Limit, err = strconv.ParseInt(limitRequested, 10, 64)
if err != nil {
return Pagination{}, err
}
}

if orderRequested, err := RequestUnaryParam(r, "order"); err != nil {
return Pagination{}, err
} else if orderRequested != "" && orderRequested == string(OrderDesc) {
paginate.Order = OrderDesc
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we have an "invalid order" error if orderRequested != "asc" or "desc"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how do you feel about the minimal approach first, having the api do some opinionated error proofing rather than more chatter? will put in the error response, but wanted to get a read on that style aspect first?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no strong opinion, agreed that we should keep things minimal to keep eyes on the prize - aka working mvp

}

return paginate, nil
}
86 changes: 32 additions & 54 deletions exp/lighthorizon/actions/operation.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package actions

import (
"encoding/json"
"fmt"
"github.com/stellar/go/support/log"
"io"
"net/http"
"net/url"
"strconv"

"github.com/stellar/go/exp/lighthorizon/adapters"
Expand All @@ -26,102 +24,82 @@ func Operations(archiveWrapper archive.Wrapper, indexStore index.Store) func(htt
return
}

query, err := url.ParseQuery(r.URL.RawQuery)
paginate, err := Paging(r)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
sendErrorResponse(w, http.StatusBadRequest, string(InvalidPagingParameters))
return
}

var cursor int64
if query.Get("cursor") == "" {
cursor = toid.New(1, 1, 1).ToInt64()
} else {
cursor, err = strconv.ParseInt(query.Get("cursor"), 10, 64)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
return
}
if paginate.Cursor < 1 {
paginate.Cursor = toid.New(1, 1, 1).ToInt64()
}

var limit int64
if query.Get("limit") == "" {
limit = 10
} else {
limit, err = strconv.ParseInt(query.Get("limit"), 10, 64)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
return
}
}

if limit == 0 || limit > 200 {
limit = 10
if paginate.Limit < 1 || paginate.Limit > 200 {
paginate.Limit = 10
}

page := hal.Page{
Cursor: query.Get("cursor"),
Order: "asc",
Limit: uint64(limit),
Cursor: strconv.FormatInt(paginate.Cursor, 10),
Order: string(paginate.Order),
Limit: uint64(paginate.Limit),
}
page.Init()
page.FullURL = r.URL

// For now, use a query param for now to avoid dragging in chi-router. Not
// really the point of the experiment yet.
account := query.Get("account")
account, err := RequestUnaryParam(r, "account")
if err != nil {
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
return
}

if account != "" {
// Skip the cursor ahead to the next active checkpoint for this account
checkpoint, err := indexStore.NextActive(account, "all_all", uint32(toid.Parse(cursor).LedgerSequence/64))
var checkpoint uint32
checkpoint, err = indexStore.NextActive(account, "all/all", uint32(toid.Parse(paginate.Cursor).LedgerSequence/64))
if err == io.EOF {
// never active. No results.
page.PopulateLinks()

encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
err = encoder.Encode(page)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
return
}
sendPageResponse(w, page)
return
} else if err != nil {
fmt.Fprintf(w, "Error: %v", err)
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
return
}
ledger := int32(checkpoint * 64)
if ledger < 0 {
// Check we don't overflow going from uint32 -> int32
fmt.Fprintf(w, "Error: Ledger overflow")
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
return
}
cursor = toid.New(ledger, 1, 1).ToInt64()
paginate.Cursor = toid.New(ledger, 1, 1).ToInt64()
}

ops, err := archiveWrapper.GetOperations(cursor, limit)
//TODO - implement paginate.Order(asc/desc)
ops, err := archiveWrapper.GetOperations(r.Context(), paginate.Cursor, paginate.Limit)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
return
}

for _, op := range ops {
var response operations.Operation
response, err = adapters.PopulateOperation(r, &op)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
return
}

page.Add(response)
}

page.PopulateLinks()

encoder := json.NewEncoder(w)
encoder.SetIndent("", " ")
err = encoder.Encode(page)
if err != nil {
fmt.Fprintf(w, "Error: %v", err)
return
}
sendPageResponse(w, page)
}
}
Loading