[feature] Implement backfilling statuses thru scheduled_at (#3685)

* Implement backfilling statuses thru scheduled_at

* Forbid mentioning others in backfills

* Update error messages & codes

* Add new tests for backfilled statuses

* Test that backfilling doesn't timeline or notify

* Fix check for absence of notification

* Test that backfills do not cause federation

* Fix type of apimodel.StatusCreateRequest.ScheduledAt in tests

* Add config file switch and min date check

docs/api/swagger.yaml

@@ -10397,10 +10397,14 @@ paths:
                   x-go-name: Federated
                 - description: |-
                     ISO 8601 Datetime at which to schedule a status.
-                    Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+
+                    Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
                     Must be at least 5 minutes in the future.
+                    This feature isn't implemented yet.
 
-                    This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+                    Providing this parameter with a *past* time will cause the status to be backdated,
+                    and will not push it to the user's followers. This is intended for importing old statuses.
+                  format: date-time
                   in: formData
                   name: scheduled_at
                   type: string

docs/configuration/instance.md

@@ -171,4 +171,17 @@ instance-subscriptions-process-every: "24h"
 # Options: ["", "zero", "serve", "baffle"]
 # Default: ""
 instance-stats-mode: ""
+
+# Bool. This flag controls whether local accounts may backdate statuses
+# using past dates with the scheduled_at param to /api/v1/statuses.
+# This flag does not affect scheduling posts in the future
+# (which is currently not implemented anyway),
+# nor can it prevent remote accounts from backdating their own statuses.
+#
+# If true, all local accounts may backdate statuses.
+# If false, status backdating will be disabled and an error will be returned if it's used.
+#
+# Options: [true, false]
+# Default: true
+instance-allow-backdating-statuses: true
 ```

example/config.yaml

@@ -458,6 +458,19 @@ instance-subscriptions-process-every: "24h"
 # Default: ""
 instance-stats-mode: ""
 
+# Bool. This flag controls whether local accounts may backdate statuses
+# using past dates with the scheduled_at param to /api/v1/statuses.
+# This flag does not affect scheduling posts in the future
+# (which is currently not implemented anyway),
+# nor can it prevent remote accounts from backdating their own statuses.
+#
+# If true, all local accounts may backdate statuses.
+# If false, status backdating will be disabled and an error will be returned if it's used.
+#
+# Options: [true, false]
+# Default: true
+instance-allow-backdating-statuses: true
+
 ###########################
 ##### ACCOUNTS CONFIG #####
 ###########################

internal/api/client/statuses/statuscreate.go

@@ -179,11 +179,15 @@ import (
 //		x-go-name: ScheduledAt
 //		description: |-
 //			ISO 8601 Datetime at which to schedule a status.
-//			Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+//
+//			Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
 //			Must be at least 5 minutes in the future.
+//			This feature isn't implemented yet.
 //
-//			This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+//			Providing this parameter with a *past* time will cause the status to be backdated,
+//			and will not push it to the user's followers. This is intended for importing old statuses.
 //		type: string
+//		format: date-time
 //		in: formData
 //	-
 //		name: language
@@ -384,12 +388,6 @@ func parseStatusCreateForm(c *gin.Context) (*apimodel.StatusCreateRequest, gtser
 		return nil, gtserror.NewErrorNotAcceptable(errors.New(text), text)
 	}
 
-	// Check not scheduled status.
-	if form.ScheduledAt != "" {
-		const text = "scheduled_at is not yet implemented"
-		return nil, gtserror.NewErrorNotImplemented(errors.New(text), text)
-	}
-
 	// Check if the deprecated "federated" field was
 	// set in lieu of "local_only", and use it if so.
 	if form.LocalOnly == nil && form.Federated != nil { // nolint:staticcheck

internal/api/client/statuses/statuscreate_test.go

@@ -20,14 +20,18 @@ package statuses_test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -41,10 +45,11 @@ const (
 	statusMarkdown         = "# Title\n\n## Smaller title\n\nThis is a post written in [markdown](https://www.markdownguide.org/)\n\n<img src=\"https://d33wubrfki0l68.cloudfront.net/f1f475a6fda1c2c4be4cac04033db5c3293032b4/513a4/assets/images/markdown-mark-white.svg\"/>"
 )
 
-func (suite *StatusCreateTestSuite) postStatus(
+// Post a status.
+func (suite *StatusCreateTestSuite) postStatusCore(
 	formData map[string][]string,
 	jsonData string,
-) (string, *httptest.ResponseRecorder) {
+) *httptest.ResponseRecorder {
 	recorder := httptest.NewRecorder()
 	ctx, _ := testrig.CreateGinTestContext(recorder, nil)
 	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
@@ -77,9 +82,42 @@ func (suite *StatusCreateTestSuite) postStatus(
 
 	// Trigger handler.
 	suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+	return recorder
+}
+
+// Post a status and return the result as deterministic JSON.
+func (suite *StatusCreateTestSuite) postStatus(
+	formData map[string][]string,
+	jsonData string,
+) (string, *httptest.ResponseRecorder) {
+	recorder := suite.postStatusCore(formData, jsonData)
 	return suite.parseStatusResponse(recorder)
 }
 
+// Post a status and return the result as a non-deterministic API structure.
+func (suite *StatusCreateTestSuite) postStatusStruct(
+	formData map[string][]string,
+	jsonData string,
+) (*apimodel.Status, *httptest.ResponseRecorder) {
+	recorder := suite.postStatusCore(formData, jsonData)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	data, err := io.ReadAll(result.Body)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	apiStatus := apimodel.Status{}
+	if err := json.Unmarshal(data, &apiStatus); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	return &apiStatus, recorder
+}
+
 // Post a new status with some custom visibility settings
 func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	out, recorder := suite.postStatus(map[string][]string{
@@ -383,10 +421,98 @@ func (suite *StatusCreateTestSuite) TestPostNewScheduledStatus() {
 
 	// We should have a helpful error message.
 	suite.Equal(`{
-  "error": "Not Implemented: scheduled_at is not yet implemented"
+  "error": "Not Implemented: scheduled statuses are not yet supported"
 }`, out)
 }
 
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatus() {
+	// A time in the past.
+	scheduledAtStr := "2020-10-04T15:32:02.018Z"
+	scheduledAt, err := time.Parse(time.RFC3339Nano, scheduledAtStr)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	status, recorder := suite.postStatusStruct(map[string][]string{
+		"status":       {"this is a recycled status from the past!"},
+		"scheduled_at": {scheduledAtStr},
+	}, "")
+
+	// Creating a status in the past should succeed.
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// The status should be backdated.
+	createdAt, err := time.Parse(time.RFC3339Nano, status.CreatedAt)
+	if err != nil {
+		suite.FailNow(err.Error())
+		return
+	}
+	suite.Equal(scheduledAt, createdAt.UTC())
+
+	// The status's ULID should be backdated.
+	timeFromULID, err := id.TimeFromULID(status.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+		return
+	}
+	suite.Equal(scheduledAt, timeFromULID.UTC())
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfMention() {
+	_, recorder := suite.postStatus(map[string][]string{
+		"status":       {"@the_mighty_zork this is a recycled mention from the past!"},
+		"scheduled_at": {"2020-10-04T15:32:02.018Z"},
+	}, "")
+
+	// Mentioning yourself is allowed in backfilled statuses.
+	suite.Equal(http.StatusOK, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithMention() {
+	_, recorder := suite.postStatus(map[string][]string{
+		"status":       {"@admin this is a recycled mention from the past!"},
+		"scheduled_at": {"2020-10-04T15:32:02.018Z"},
+	}, "")
+
+	// Mentioning others is forbidden in backfilled statuses.
+	suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfReply() {
+	_, recorder := suite.postStatus(map[string][]string{
+		"status":         {"this is a recycled reply from the past!"},
+		"scheduled_at":   {"2020-10-04T15:32:02.018Z"},
+		"in_reply_to_id": {suite.testStatuses["local_account_1_status_1"].ID},
+	}, "")
+
+	// Replying to yourself is allowed in backfilled statuses.
+	suite.Equal(http.StatusOK, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithReply() {
+	_, recorder := suite.postStatus(map[string][]string{
+		"status":         {"this is a recycled reply from the past!"},
+		"scheduled_at":   {"2020-10-04T15:32:02.018Z"},
+		"in_reply_to_id": {suite.testStatuses["admin_account_status_1"].ID},
+	}, "")
+
+	// Replying to others is forbidden in backfilled statuses.
+	suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithPoll() {
+	_, recorder := suite.postStatus(map[string][]string{
+		"status":           {"this is a recycled poll from the past!"},
+		"scheduled_at":     {"2020-10-04T15:32:02.018Z"},
+		"poll[options][]":  {"first option", "second option"},
+		"poll[expires_in]": {"3600"},
+		"poll[multiple]":   {"true"},
+	}, "")
+
+	// Polls are forbidden in backfilled statuses.
+	suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
 func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() {
 	out, recorder := suite.postStatus(map[string][]string{
 		"status":       {statusMarkdown},

internal/api/model/status.go

@@ -17,7 +17,11 @@
 
 package model
 
-import "github.com/superseriousbusiness/gotosocial/internal/language"
+import (
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/language"
+)
 
 // Status models a status or post.
 //
@@ -231,9 +235,14 @@ type StatusCreateRequest struct {
 	Federated *bool `form:"federated" json:"federated"`
 
 	// ISO 8601 Datetime at which to schedule a status.
-	// Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+	//
+	// Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
 	// Must be at least 5 minutes in the future.
-	ScheduledAt string `form:"scheduled_at" json:"scheduled_at"`
+	// This feature isn't implemented yet.
+	//
+	// Providing this parameter with a *past* time will cause the status to be backdated,
+	// and will not push it to the user's followers. This is intended for importing old statuses.
+	ScheduledAt *time.Time `form:"scheduled_at" json:"scheduled_at"`
 
 	// ISO 639 language code for this status.
 	Language string `form:"language" json:"language"`

internal/config/config.go

@@ -91,6 +91,7 @@ type Configuration struct {
 	InstanceSubscriptionsProcessFrom  string             `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
 	InstanceSubscriptionsProcessEvery time.Duration      `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
 	InstanceStatsMode                 string             `name:"instance-stats-mode" usage:"Allows you to customize the way stats are served to crawlers: one of '', 'serve', 'zero', 'baffle'. Home page stats remain unchanged."`
+	InstanceAllowBackdatingStatuses   bool               `name:"instance-allow-backdating-statuses" usage:"Allow local accounts to backdate statuses using the scheduled_at param to /api/v1/statuses"`
 
 	AccountsRegistrationOpen         bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
 	AccountsReasonRequired           bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`

internal/config/defaults.go

@@ -67,6 +67,7 @@ var Defaults = Configuration{
 	InstanceLanguages:                 make(language.Languages, 0),
 	InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
 	InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
+	InstanceAllowBackdatingStatuses:   true,
 
 	AccountsRegistrationOpen:         false,
 	AccountsReasonRequired:           true,

internal/config/flags.go

@@ -93,6 +93,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage"))
 		cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage"))
 		cmd.Flags().String(InstanceStatsModeFlag(), cfg.InstanceStatsMode, fieldtag("InstanceStatsMode", "usage"))
+		cmd.Flags().Bool(InstanceAllowBackdatingStatusesFlag(), cfg.InstanceAllowBackdatingStatuses, fieldtag("InstanceAllowBackdatingStatuses", "usage"))
 
 		// Accounts
 		cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage"))

internal/config/helpers.gen.go

@@ -1082,6 +1082,31 @@ func GetInstanceStatsMode() string { return global.GetInstanceStatsMode() }
 // SetInstanceStatsMode safely sets the value for global configuration 'InstanceStatsMode' field
 func SetInstanceStatsMode(v string) { global.SetInstanceStatsMode(v) }
 
+// GetInstanceAllowBackdatingStatuses safely fetches the Configuration value for state's 'InstanceAllowBackdatingStatuses' field
+func (st *ConfigState) GetInstanceAllowBackdatingStatuses() (v bool) {
+	st.mutex.RLock()
+	v = st.config.InstanceAllowBackdatingStatuses
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceAllowBackdatingStatuses safely sets the Configuration value for state's 'InstanceAllowBackdatingStatuses' field
+func (st *ConfigState) SetInstanceAllowBackdatingStatuses(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceAllowBackdatingStatuses = v
+	st.reloadToViper()
+}
+
+// InstanceAllowBackdatingStatusesFlag returns the flag name for the 'InstanceAllowBackdatingStatuses' field
+func InstanceAllowBackdatingStatusesFlag() string { return "instance-allow-backdating-statuses" }
+
+// GetInstanceAllowBackdatingStatuses safely fetches the value for global configuration 'InstanceAllowBackdatingStatuses' field
+func GetInstanceAllowBackdatingStatuses() bool { return global.GetInstanceAllowBackdatingStatuses() }
+
+// SetInstanceAllowBackdatingStatuses safely sets the value for global configuration 'InstanceAllowBackdatingStatuses' field
+func SetInstanceAllowBackdatingStatuses(v bool) { global.SetInstanceAllowBackdatingStatuses(v) }
+
 // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
 func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
 	st.mutex.RLock()

internal/federation/dereferencing/instance_test.go

@@ -50,7 +50,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
 			//
 			// Debug-level logs should show something like:
 			//
-			//   - "can't fetch /nodeinfo/2.1: robots.txt disallows it" 
+			//   - "can't fetch /nodeinfo/2.1: robots.txt disallows it"
 			instanceIRI:      testrig.URLMustParse("https://furtive-nerds.example.org"),
 			expectedSoftware: "",
 		},
@@ -60,7 +60,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
 			//
 			// Debug-level logs should show something like:
 			//
-			//   - "can't fetch api/v1/instance: robots.txt disallows it" 
+			//   - "can't fetch api/v1/instance: robots.txt disallows it"
 			//   - "can't fetch .well-known/nodeinfo: robots.txt disallows it"
 			instanceIRI:      testrig.URLMustParse("https://robotic.furtive-nerds.example.org"),
 			expectedSoftware: "",
@@ -71,7 +71,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
 			//
 			// Debug-level logs should show something like:
 			//
-			//   - "can't use fetched .well-known/nodeinfo: robots tags disallows it" 
+			//   - "can't use fetched .well-known/nodeinfo: robots tags disallows it"
 			instanceIRI:      testrig.URLMustParse("https://really.furtive-nerds.example.org"),
 			expectedSoftware: "",
 		},