feat(server): export items as geojson and csv via public api

server/e2e/publicapi_test.go

@@ -29,16 +29,21 @@ var (
 	publicAPIItem2ID      = id.NewItemID()
 	publicAPIItem3ID      = id.NewItemID()
 	publicAPIItem4ID      = id.NewItemID()
+	publicAPIItem6ID      = id.NewItemID()
+	publicAPIItem7ID      = id.NewItemID()
 	publicAPIAsset1ID     = id.NewAssetID()
 	publicAPIAsset2ID     = id.NewAssetID()
 	publicAPIAssetUUID    = uuid.NewString()
 	publicAPIProjectAlias = "test-project"
 	publicAPIModelKey     = "test-model"
 	publicAPIModelKey2    = "test-model-2"
+	publicAPIModelKey3    = "test-model-3"
 	publicAPIField1Key    = "test-field-1"
 	publicAPIField2Key    = "asset"
 	publicAPIField3Key    = "test-field-2"
 	publicAPIField4Key    = "asset2"
+	publicAPIField5Key    = "geometry-object"
+	publicAPIField6Key    = "geometry-editor"
 )
 
 func TestPublicAPI(t *testing.T) {
@@ -103,8 +108,26 @@ func TestPublicAPI(t *testing.T) {
 						},
 					},
 				},
+				{
+					"id":               publicAPIItem6ID.String(),
+					publicAPIField1Key: "ccc",
+					publicAPIField3Key: []string{"aaa", "bbb", "ccc"},
+					publicAPIField4Key: []any{
+						map[string]any{
+							"type": "asset",
+							"id":   publicAPIAsset1ID.String(),
+							"url":  fmt.Sprintf("https://example.com/assets/%s/%s/aaa.zip", publicAPIAssetUUID[:2], publicAPIAssetUUID[2:]),
+						},
+					},
+					publicAPIField5Key: "{\n\"type\": \"Point\",\n\t\"coordinates\": [102.0, 0.5]\n}",
+					publicAPIField6Key: "{\"coordinates\":[[139.65439725962517,36.34793305387103],[139.61688622815393,35.910803456352724]],\"type\":\"LineString\"}",
+				},
+				{
+					"id":               publicAPIItem7ID.String(),
+					publicAPIField1Key: "ccc",
+				},
 			},
-			"totalCount": 3,
+			"totalCount": 5,
 			"hasMore":    false,
 			"limit":      50,
 			"offset":     0,
@@ -125,8 +148,8 @@ func TestPublicAPI(t *testing.T) {
 					publicAPIField1Key: "bbb",
 				},
 			},
-			"totalCount": 3,
-			"hasMore":    false,
+			"totalCount": 5,
+			"hasMore":    true,
 			"limit":      1,
 			"offset":     1,
 			"page":       2,
@@ -146,7 +169,7 @@ func TestPublicAPI(t *testing.T) {
 					publicAPIField1Key: "bbb",
 				},
 			},
-			"totalCount": 3,
+			"totalCount": 5,
 			"hasMore":    true,
 			"nextCursor": publicAPIItem2ID.String(),
 		})
@@ -221,14 +244,78 @@ func TestPublicAPI(t *testing.T) {
 					publicAPIField3Key: []string{"aaa", "bbb", "ccc"},
 					// publicAPIField4Key should be removed
 				},
+				{
+					"id":               publicAPIItem6ID.String(),
+					publicAPIField1Key: "ccc",
+					publicAPIField3Key: []string{"aaa", "bbb", "ccc"},
+					publicAPIField5Key: "{\n\"type\": \"Point\",\n\t\"coordinates\": [102.0, 0.5]\n}",
+					publicAPIField6Key: "{\"coordinates\":[[139.65439725962517,36.34793305387103],[139.61688622815393,35.910803456352724]],\"type\":\"LineString\"}",
+				},
+				{
+					"id":               publicAPIItem7ID.String(),
+					publicAPIField1Key: "ccc",
+				},
 			},
-			"totalCount": 3,
+			"totalCount": 5,
 			"hasMore":    false,
 			"limit":      50,
 			"offset":     0,
 			"page":       1,
 		})
 
+	e.GET("/api/p/{project}/{model}.geojson", publicAPIProjectAlias, publicAPIModelKey).
+		Expect().
+		Status(http.StatusOK).
+		JSON().
+		IsEqual(map[string]interface{}{
+			"type": "FeatureCollection",
+			"features": []map[string]interface{}{
+				{
+					"type": "Feature",
+					"id":   publicAPIItem6ID.String(),
+					"geometry": map[string]interface{}{
+						"type": "Point",
+						"coordinates": []interface{}{
+							102,
+							0.5,
+						},
+					},
+					"properties": map[string]interface{}{
+						"test-field-1": "ccc",
+						"test-field-2": []interface{}{
+							"aaa",
+							"bbb",
+							"ccc",
+						},
+					},
+				},
+			},
+		})
+
+	// no geometry field
+	e.GET("/api/p/{project}/{model}.geojson", publicAPIProjectAlias, publicAPIModelKey3).
+		Expect().
+		Status(http.StatusNotFound).
+		JSON().
+		IsEqual(map[string]interface{}{
+			"error": "not found",
+		})
+
+	e.GET("/api/p/{project}/{model}.csv", publicAPIProjectAlias, publicAPIModelKey).
+		Expect().
+		Status(http.StatusOK).
+		Body().
+		IsEqual(fmt.Sprintf("id,location_lat,location_lng,test-field-1,asset,test-field-2,asset2\n%s,102,0.5,ccc,,aaa,\n", publicAPIItem6ID.String()))
+
+	// no geometry field
+	e.GET("/api/p/{project}/{model}.csv", publicAPIProjectAlias, publicAPIModelKey3).
+		Expect().
+		Status(http.StatusNotFound).
+		JSON().
+		IsEqual(map[string]interface{}{
+			"error": "not found",
+		})
+
 	e.GET("/api/p/{project}/{model}/{item}", publicAPIProjectAlias, publicAPIModelKey, publicAPIItem1ID).
 		Expect().
 		Status(http.StatusOK).
@@ -271,16 +358,25 @@ func publicAPISeeder(ctx context.Context, r *repo.Container) error {
 	af := asset.NewFile().Name("bbb.txt").Path("aaa/bbb.txt").Build()
 
 	fid := id.NewFieldID()
+	gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString}
+	gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString}
 	s := schema.New().NewID().Project(p1.ID()).Workspace(p1.Workspace()).Fields(schema.FieldList{
-		schema.NewField(schema.NewText(nil).TypeProperty()).ID(fid).Key(key.New(publicAPIField1Key)).MustBuild(),
-		schema.NewField(schema.NewAsset().TypeProperty()).NewID().Key(key.New(publicAPIField2Key)).MustBuild(),
-		schema.NewField(schema.NewText(nil).TypeProperty()).NewID().Key(key.New(publicAPIField3Key)).Multiple(true).MustBuild(),
-		schema.NewField(schema.NewAsset().TypeProperty()).NewID().Key(key.New(publicAPIField4Key)).Multiple(true).MustBuild(),
+		schema.NewField(schema.NewText(nil).TypeProperty()).ID(fid).Name(publicAPIField1Key).Key(key.New(publicAPIField1Key)).MustBuild(),
+		schema.NewField(schema.NewAsset().TypeProperty()).NewID().Name(publicAPIField2Key).Key(key.New(publicAPIField2Key)).MustBuild(),
+		schema.NewField(schema.NewText(nil).TypeProperty()).NewID().Name(publicAPIField3Key).Key(key.New(publicAPIField3Key)).Multiple(true).MustBuild(),
+		schema.NewField(schema.NewAsset().TypeProperty()).NewID().Name(publicAPIField4Key).Key(key.New(publicAPIField4Key)).Multiple(true).MustBuild(),
+		schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name(publicAPIField5Key).Key(key.New(publicAPIField5Key)).MustBuild(),
+		schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name(publicAPIField6Key).Key(key.New(publicAPIField6Key)).MustBuild(),
 	}).TitleField(fid.Ref()).MustBuild()
 
+	s2 := schema.New().NewID().Project(p1.ID()).Workspace(p1.Workspace()).Fields(schema.FieldList{
+		schema.NewField(schema.NewText(nil).TypeProperty()).ID(fid).Name(publicAPIField1Key).Key(key.New(publicAPIField1Key)).MustBuild(),
+	}).MustBuild()
+
 	m := model.New().ID(publicAPIModelID).Project(p1.ID()).Schema(s.ID()).Public(true).Key(key.New(publicAPIModelKey)).MustBuild()
-	// not public model
-	m2 := model.New().ID(publicAPIModelID).Project(p1.ID()).Schema(s.ID()).Key(key.New(publicAPIModelKey2)).Public(false).MustBuild()
+	// m2 is not a public model
+	m2 := model.New().ID(publicAPIModelID).Project(p1.ID()).Schema(s.ID()).Name(publicAPIModelKey2).Key(key.New(publicAPIModelKey2)).Public(false).MustBuild()
+	m3 := model.New().ID(publicAPIModelID).Project(p1.ID()).Schema(s2.ID()).Name(publicAPIModelKey3).Key(key.New(publicAPIModelKey3)).Public(true).MustBuild()
 
 	i1 := item.New().ID(publicAPIItem1ID).Model(m.ID()).Schema(s.ID()).Project(p1.ID()).Thread(id.NewThreadID()).User(uid).Fields([]*item.Field{
 		item.NewField(s.Fields()[0].ID(), value.TypeText.Value("aaa").AsMultiple(), nil),
@@ -308,6 +404,19 @@ func publicAPISeeder(ctx context.Context, r *repo.Container) error {
 		item.NewField(s.Fields()[1].ID(), value.TypeAsset.Value(a.ID()).AsMultiple(), nil),
 	}).MustBuild()
 
+	i6 := item.New().ID(publicAPIItem6ID).Model(m.ID()).Schema(s.ID()).Project(p1.ID()).Thread(id.NewThreadID()).User(uid).Fields([]*item.Field{
+		item.NewField(s.Fields()[0].ID(), value.TypeText.Value("ccc").AsMultiple(), nil),
+		item.NewField(s.Fields()[1].ID(), value.TypeAsset.Value(publicAPIAsset2ID).AsMultiple(), nil),
+		item.NewField(s.Fields()[2].ID(), value.NewMultiple(value.TypeText, []any{"aaa", "bbb", "ccc"}), nil),
+		item.NewField(s.Fields()[3].ID(), value.TypeAsset.Value(a.ID()).AsMultiple(), nil),
+		item.NewField(s.Fields()[4].ID(), value.TypeGeometryObject.Value("{\n\"type\": \"Point\",\n\t\"coordinates\": [102.0, 0.5]\n}").AsMultiple(), nil),
+		item.NewField(s.Fields()[5].ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[[139.65439725962517,36.34793305387103],[139.61688622815393,35.910803456352724]],\"type\":\"LineString\"}").AsMultiple(), nil),
+	}).MustBuild()
+
+	i7 := item.New().ID(publicAPIItem7ID).Model(m3.ID()).Schema(s2.ID()).Project(p1.ID()).Thread(id.NewThreadID()).User(uid).Fields([]*item.Field{
+		item.NewField(s.Fields()[0].ID(), value.TypeText.Value("ccc").AsMultiple(), nil),
+	}).MustBuild()
+
 	lo.Must0(r.Project.Save(ctx, p1))
 	lo.Must0(r.Asset.Save(ctx, a))
 	lo.Must0(r.AssetFile.Save(ctx, a.ID(), af))
@@ -318,9 +427,13 @@ func publicAPISeeder(ctx context.Context, r *repo.Container) error {
 	lo.Must0(r.Item.Save(ctx, i3))
 	lo.Must0(r.Item.Save(ctx, i4))
 	lo.Must0(r.Item.Save(ctx, i5))
+	lo.Must0(r.Item.Save(ctx, i6))
+	lo.Must0(r.Item.Save(ctx, i7))
 	lo.Must0(r.Item.UpdateRef(ctx, i1.ID(), version.Public, version.Latest.OrVersion().Ref()))
 	lo.Must0(r.Item.UpdateRef(ctx, i2.ID(), version.Public, version.Latest.OrVersion().Ref()))
 	lo.Must0(r.Item.UpdateRef(ctx, i3.ID(), version.Public, version.Latest.OrVersion().Ref()))
+	lo.Must0(r.Item.UpdateRef(ctx, i6.ID(), version.Public, version.Latest.OrVersion().Ref()))
+	lo.Must0(r.Item.UpdateRef(ctx, i7.ID(), version.Public, version.Latest.OrVersion().Ref()))
 
 	return nil
 }

server/internal/adapter/integration/item.go

@@ -3,7 +3,7 @@ package integration
 import (
 	"context"
 	"errors"
-	"strings"
+	"io"
 
 	"github.com/reearth/reearth-cms/server/internal/usecase"
 	"github.com/reearth/reearth-cms/server/pkg/model"
@@ -117,15 +117,13 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject
 		return ItemsAsCSV400Response{}, err
 	}
 
-	csvString, err := integrationapi.CSVFromItems(items, sp.Schema())
+	pr, pw := io.Pipe()
+	err = integrationapi.CSVFromItems(pw, items, sp.Schema())
 	if err != nil {
 		return nil, err
 	}
-	reader := strings.NewReader(csvString)
-	contentLength := reader.Len()
 	return ItemsAsCSV200TextcsvResponse{
-		Body:          reader,
-		ContentLength: int64(contentLength),
+		Body: pr,
 	}, nil
 }
 
@@ -277,15 +275,13 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro
 		return ItemsWithProjectAsCSV400Response{}, err
 	}
 
-	csvString, err := integrationapi.CSVFromItems(items, sp.Schema())
+	pr, pw := io.Pipe()
+	err = integrationapi.CSVFromItems(pw, items, sp.Schema())
 	if err != nil {
 		return nil, err
 	}
-	reader := strings.NewReader(csvString)
-	contentLength := reader.Len()
 	return ItemsWithProjectAsCSV200TextcsvResponse{
-		Body:          reader,
-		ContentLength: int64(contentLength),
+		Body: pr,
 	}, nil
 }
 

server/internal/adapter/publicapi/api.go

@@ -2,18 +2,12 @@ package publicapi
 
 import (
 	"context"
-	"encoding/csv"
-	"encoding/json"
-	"fmt"
-	"io"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	"github.com/reearth/reearth-cms/server/pkg/schema"
-	"github.com/reearth/reearthx/log"
 	"github.com/reearth/reearthx/usecasex"
 	"github.com/samber/lo"
 )
@@ -74,22 +68,29 @@ func PublicApiItemList() echo.HandlerFunc {
 		if strings.Contains(m, ".") {
 			m, resType, _ = strings.Cut(m, ".")
 		}
-		if resType != "csv" && resType != "json" {
+		if resType != "csv" && resType != "json" && resType != "geojson" {
 			resType = "json"
 		}
 
-		res, s, err := ctrl.GetItems(ctx, c.Param("project"), m, p)
+		items, _, err := ctrl.GetItems(ctx, c.Param("project"), m, p)
 		if err != nil {
 			return err
 		}
 
+		vi, s, err1 := ctrl.GetVersionedItems(ctx, c.Param("project"), m, p)
+		if err1 != nil {
+			return err1
+		}
+
 		switch resType {
 		case "csv":
-			return toCSV(c, res, s)
+			return toCSV(c, vi, s)
+		case "geojson":
+			return toGeoJSON(c, vi, s)
 		case "json":
-			return c.JSON(http.StatusOK, res)
+			return c.JSON(http.StatusOK, items)
 		default:
-			return c.JSON(http.StatusOK, res)
+			return c.JSON(http.StatusOK, items)
 		}
 	}
 }
@@ -157,46 +158,3 @@ func intParams(c echo.Context, params ...string) (int64, bool) {
 	}
 	return 0, false
 }
-
-func toCSV(c echo.Context, l ListResult[Item], s *schema.Schema) error {
-	pr, pw := io.Pipe()
-
-	go func() {
-		var err error
-		defer func() {
-			_ = pw.CloseWithError(err)
-		}()
-
-		w := csv.NewWriter(pw)
-		keys := lo.Map(s.Fields(), func(f *schema.Field, _ int) string {
-			return f.Key().String()
-		})
-		err = w.Write(append([]string{"id"}, keys...))
-		if err != nil {
-			log.Errorf("filed to write csv headers, err: %+v", err)
-			return
-		}
-
-		for _, itm := range l.Results {
-			values := []string{itm.ID}
-			for _, k := range keys {
-				// values = append(values, fmt.Sprintf("%v", itm.Fields[k]))
-				var v []byte
-				v, err = json.Marshal(itm.Fields[k])
-				if err != nil {
-					log.Errorf("filed to json marshal field value, err: %+v", err)
-					return
-				}
-				values = append(values, fmt.Sprintf("%v", string(v)))
-			}
-			err = w.Write(values)
-			if err != nil {
-				log.Errorf("filed to write csv value, err: %+v", err)
-				return
-			}
-		}
-		w.Flush()
-	}()
-
-	return c.Stream(http.StatusOK, "text/csv", pr)
-}