Allow full BCP 47 in language inputs

internal/processing/account/create.go

@@ -28,6 +28,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
+	"github.com/superseriousbusiness/gotosocial/internal/validate/normalize"
 	"github.com/superseriousbusiness/oauth2/v4"
 )
 
@@ -75,7 +76,7 @@ func (p *Processor) Create(
 		Reason:      text.SanitizePlaintext(reason),
 		PreApproved: !config.GetAccountsApprovalRequired(), // Mark as approved if no approval required.
 		SignUpIP:    form.IP,
-		Locale:      form.Locale,
+		Locale:      normalize.Language(form.Locale),
 		AppID:       app.ID,
 	})
 	if err != nil {

internal/processing/account/update.go

@@ -34,6 +34,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/validate"
+	"github.com/superseriousbusiness/gotosocial/internal/validate/normalize"
 )
 
 func (p *Processor) selectNoteFormatter(contentType string) text.FormatFunc {
@@ -225,7 +226,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
 			if err := validate.Language(*form.Source.Language); err != nil {
 				return nil, gtserror.NewErrorBadRequest(err)
 			}
-			account.Language = *form.Source.Language
+			account.Language = normalize.Language(*form.Source.Language)
 		}
 
 		if form.Source.Sensitive != nil {

internal/processing/status/create.go

@@ -34,6 +34,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
+	"github.com/superseriousbusiness/gotosocial/internal/validate/normalize"
 )
 
 // Create processes the given form to create a new status, returning the api model representation of that status if it's OK.
@@ -55,7 +56,6 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, appli
 		ContentWarning:           text.SanitizePlaintext(form.SpoilerText),
 		ActivityStreamsType:      ap.ObjectNote,
 		Sensitive:                &sensitive,
-		Language:                 form.Language,
 		CreatedWithApplicationID: application.ID,
 		Text:                     form.Status,
 	}
@@ -266,7 +266,7 @@ func processVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateF
 
 func processLanguage(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
 	if form.Language != "" {
-		status.Language = form.Language
+		status.Language = normalize.Language(form.Language)
 	} else {
 		status.Language = accountDefaultLanguage
 	}

internal/processing/status/create_test.go

@@ -208,6 +208,74 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() {
 	suite.Nil(apiStatus)
 }
 
+func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() {
+	ctx := context.Background()
+
+	creatingAccount := suite.testAccounts["local_account_1"]
+	creatingApplication := suite.testApplications["application_1"]
+
+	statusCreateForm := &apimodel.AdvancedStatusCreateForm{
+		StatusCreateRequest: apimodel.StatusCreateRequest{
+			Status:      "你好世界", // hello world
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  apimodel.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "zh-Hans",
+			ContentType: apimodel.StatusContentTypePlain,
+		},
+		AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
+	suite.NoError(err)
+	suite.NotNil(apiStatus)
+
+	suite.Equal("zh-Hans", *apiStatus.Language)
+}
+
+func (suite *StatusCreateTestSuite) TestProcessLanguageWithNoncanonicalLanguageTag() {
+	ctx := context.Background()
+
+	creatingAccount := suite.testAccounts["local_account_1"]
+	creatingApplication := suite.testApplications["application_1"]
+
+	statusCreateForm := &apimodel.AdvancedStatusCreateForm{
+		StatusCreateRequest: apimodel.StatusCreateRequest{
+			Status:      "hello world",
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  apimodel.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en-us",
+			ContentType: apimodel.StatusContentTypePlain,
+		},
+		AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
+	suite.NoError(err)
+	suite.NotNil(apiStatus)
+
+	suite.Equal("en-US", *apiStatus.Language)
+}
+
 func TestStatusCreateTestSuite(t *testing.T) {
 	suite.Run(t, new(StatusCreateTestSuite))
 }

internal/validate/formvalidation.go

@@ -99,13 +99,16 @@ func Email(email string) error {
 	return err
 }
 
-// Language checks that the given language string is a 2- or 3-letter ISO 639 code.
-// Returns an error if the language cannot be parsed. See: https://pkg.go.dev/golang.org/x/text/language
+// Language checks that the given language string is a valid, if not necessarily canonical, BCP 47 language tag.
+// Returns an error if the language cannot be parsed.
+// Does not return an error if the language is not in the canonical tag format.
+// See: https://pkg.go.dev/golang.org/x/text/language
+// See: [internal/validate/normalize.Language]
 func Language(lang string) error {
 	if lang == "" {
 		return errors.New("no language provided")
 	}
-	_, err := language.ParseBase(lang)
+	_, err := language.Parse(lang)
 	return err
 }
 

internal/validate/formvalidation_test.go

@@ -168,6 +168,9 @@ func (suite *ValidationTestSuite) TestValidateLanguage() {
 	englishUS := "en-us"
 	dutch := "nl"
 	german := "de"
+	chinese := "zh"
+	chineseSimplified := "zh-Hans"
+	chineseTraditional := "zh-Hant"
 	var err error
 
 	err = validate.Language(empty)
@@ -201,8 +204,8 @@ func (suite *ValidationTestSuite) TestValidateLanguage() {
 	}
 
 	err = validate.Language(englishUS)
-	if suite.Error(err) {
-		suite.Equal(errors.New("language: tag is not well-formed"), err)
+	if suite.NoError(err) {
+		suite.Equal(nil, err)
 	}
 
 	err = validate.Language(dutch)
@@ -214,6 +217,21 @@ func (suite *ValidationTestSuite) TestValidateLanguage() {
 	if suite.NoError(err) {
 		suite.Equal(nil, err)
 	}
+
+	err = validate.Language(chinese)
+	if suite.NoError(err) {
+		suite.Equal(nil, err)
+	}
+
+	err = validate.Language(chineseSimplified)
+	if suite.NoError(err) {
+		suite.Equal(nil, err)
+	}
+
+	err = validate.Language(chineseTraditional)
+	if suite.NoError(err) {
+		suite.Equal(nil, err)
+	}
 }
 
 func (suite *ValidationTestSuite) TestValidateReason() {

internal/validate/normalize/language.go

@@ -0,0 +1,34 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors [email protected]
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package normalize
+
+import (
+	"golang.org/x/text/language"
+)
+
+// Language converts a previously validated but possibly non-canonical BCP 47 tag to its canonical form.
+// See: https://pkg.go.dev/golang.org/x/text/language
+// See: [internal/validate.Language]
+func Language(lang string) string {
+	canonical, err := language.Parse(lang)
+	if err != nil {
+		// Should not happen: input should have been previously validated.
+		return ""
+	}
+	return canonical.String()
+}

internal/validate/normalize/language_test.go

@@ -0,0 +1,89 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors [email protected]
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package normalize
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type NormalizationTestSuite struct {
+	suite.Suite
+}
+
+func (suite *NormalizationTestSuite) TestNormalizeLanguage() {
+	empty := ""
+	notALanguage := "this isn't a language at all!"
+	english := "en"
+	// Should be all lowercase
+	capitalEnglish := "EN"
+	// Overlong, should be in ISO 639-1 format
+	arabic3Letters := "ara"
+	// Should be all lowercase
+	mixedCapsEnglish := "eN"
+	// Region should be capitalized
+	englishUS := "en-us"
+	dutch := "nl"
+	german := "de"
+	chinese := "zh"
+	chineseSimplified := "zh-Hans"
+	chineseTraditional := "zh-Hant"
+
+	var actual string
+
+	actual = Language(empty)
+	suite.Equal(empty, actual)
+
+	actual = Language(notALanguage)
+	suite.Equal(empty, actual)
+
+	actual = Language(english)
+	suite.Equal(english, actual)
+
+	actual = Language(capitalEnglish)
+	suite.Equal(english, actual)
+
+	actual = Language(arabic3Letters)
+	suite.Equal("ar", actual)
+
+	actual = Language(mixedCapsEnglish)
+	suite.Equal(english, actual)
+
+	actual = Language(englishUS)
+	suite.Equal("en-US", actual)
+
+	actual = Language(dutch)
+	suite.Equal(dutch, actual)
+
+	actual = Language(german)
+	suite.Equal(german, actual)
+
+	actual = Language(chinese)
+	suite.Equal(chinese, actual)
+
+	actual = Language(chineseSimplified)
+	suite.Equal(chineseSimplified, actual)
+
+	actual = Language(chineseTraditional)
+	suite.Equal(chineseTraditional, actual)
+}
+
+func TestNormalizationTestSuite(t *testing.T) {
+	suite.Run(t, new(NormalizationTestSuite))
+}