From 89c0416a357f65d77a11b3ec5b4de53f4bf93c9e Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 3 Aug 2023 11:33:22 -0700 Subject: [PATCH 1/4] Allow full BCP 47 in language inputs Fixes #2066 --- internal/processing/account/create.go | 3 +- internal/processing/account/update.go | 3 +- internal/processing/status/create.go | 4 +- internal/processing/status/create_test.go | 68 +++++++++++++++ internal/validate/formvalidation.go | 9 +- internal/validate/formvalidation_test.go | 22 ++++- internal/validate/normalize/language.go | 34 ++++++++ internal/validate/normalize/language_test.go | 89 ++++++++++++++++++++ 8 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 internal/validate/normalize/language.go create mode 100644 internal/validate/normalize/language_test.go diff --git a/internal/processing/account/create.go b/internal/processing/account/create.go index 1a172b865c..98deddd687 100644 --- a/internal/processing/account/create.go +++ b/internal/processing/account/create.go @@ -28,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" + "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" "github.com/superseriousbusiness/oauth2/v4" ) @@ -75,7 +76,7 @@ func (p *Processor) Create( Reason: text.SanitizePlaintext(reason), PreApproved: !config.GetAccountsApprovalRequired(), // Mark as approved if no approval required. SignUpIP: form.IP, - Locale: form.Locale, + Locale: normalize.Language(form.Locale), AppID: app.ID, }) if err != nil { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 01c62d7e36..3fa058ec36 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -34,6 +34,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/validate" + "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" ) func (p *Processor) selectNoteFormatter(contentType string) text.FormatFunc { @@ -225,7 +226,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form if err := validate.Language(*form.Source.Language); err != nil { return nil, gtserror.NewErrorBadRequest(err) } - account.Language = *form.Source.Language + account.Language = normalize.Language(*form.Source.Language) } if form.Source.Sensitive != nil { diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index 2d9c3a1969..68ef76cf2d 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -34,6 +34,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/uris" + "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" ) // Create processes the given form to create a new status, returning the api model representation of that status if it's OK. @@ -55,7 +56,6 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, appli ContentWarning: text.SanitizePlaintext(form.SpoilerText), ActivityStreamsType: ap.ObjectNote, Sensitive: &sensitive, - Language: form.Language, CreatedWithApplicationID: application.ID, Text: form.Status, } @@ -266,7 +266,7 @@ func processVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateF func processLanguage(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error { if form.Language != "" { - status.Language = form.Language + status.Language = normalize.Language(form.Language) } else { status.Language = accountDefaultLanguage } diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go index 2a797516d9..701dbf3e80 100644 --- a/internal/processing/status/create_test.go +++ b/internal/processing/status/create_test.go @@ -208,6 +208,74 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() { suite.Nil(apiStatus) } +func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() { + ctx := context.Background() + + creatingAccount := suite.testAccounts["local_account_1"] + creatingApplication := suite.testApplications["application_1"] + + statusCreateForm := &apimodel.AdvancedStatusCreateForm{ + StatusCreateRequest: apimodel.StatusCreateRequest{ + Status: "你好世界", // hello world + MediaIDs: []string{}, + Poll: nil, + InReplyToID: "", + Sensitive: false, + SpoilerText: "", + Visibility: apimodel.VisibilityPublic, + ScheduledAt: "", + Language: "zh-Hans", + ContentType: apimodel.StatusContentTypePlain, + }, + AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{ + Federated: nil, + Boostable: nil, + Replyable: nil, + Likeable: nil, + }, + } + + apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm) + suite.NoError(err) + suite.NotNil(apiStatus) + + suite.Equal("zh-Hans", *apiStatus.Language) +} + +func (suite *StatusCreateTestSuite) TestProcessLanguageWithNoncanonicalLanguageTag() { + ctx := context.Background() + + creatingAccount := suite.testAccounts["local_account_1"] + creatingApplication := suite.testApplications["application_1"] + + statusCreateForm := &apimodel.AdvancedStatusCreateForm{ + StatusCreateRequest: apimodel.StatusCreateRequest{ + Status: "hello world", + MediaIDs: []string{}, + Poll: nil, + InReplyToID: "", + Sensitive: false, + SpoilerText: "", + Visibility: apimodel.VisibilityPublic, + ScheduledAt: "", + Language: "en-us", + ContentType: apimodel.StatusContentTypePlain, + }, + AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{ + Federated: nil, + Boostable: nil, + Replyable: nil, + Likeable: nil, + }, + } + + apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm) + suite.NoError(err) + suite.NotNil(apiStatus) + + suite.Equal("en-US", *apiStatus.Language) +} + func TestStatusCreateTestSuite(t *testing.T) { suite.Run(t, new(StatusCreateTestSuite)) } diff --git a/internal/validate/formvalidation.go b/internal/validate/formvalidation.go index 76ce6a8de5..3bb50a543c 100644 --- a/internal/validate/formvalidation.go +++ b/internal/validate/formvalidation.go @@ -99,13 +99,16 @@ func Email(email string) error { return err } -// Language checks that the given language string is a 2- or 3-letter ISO 639 code. -// Returns an error if the language cannot be parsed. See: https://pkg.go.dev/golang.org/x/text/language +// Language checks that the given language string is a valid, if not necessarily canonical, BCP 47 language tag. +// Returns an error if the language cannot be parsed. +// Does not return an error if the language is not in the canonical tag format. +// See: https://pkg.go.dev/golang.org/x/text/language +// See: [internal/validate/normalize.Language] func Language(lang string) error { if lang == "" { return errors.New("no language provided") } - _, err := language.ParseBase(lang) + _, err := language.Parse(lang) return err } diff --git a/internal/validate/formvalidation_test.go b/internal/validate/formvalidation_test.go index 534e5b8497..32e6c30273 100644 --- a/internal/validate/formvalidation_test.go +++ b/internal/validate/formvalidation_test.go @@ -168,6 +168,9 @@ func (suite *ValidationTestSuite) TestValidateLanguage() { englishUS := "en-us" dutch := "nl" german := "de" + chinese := "zh" + chineseSimplified := "zh-Hans" + chineseTraditional := "zh-Hant" var err error err = validate.Language(empty) @@ -201,8 +204,8 @@ func (suite *ValidationTestSuite) TestValidateLanguage() { } err = validate.Language(englishUS) - if suite.Error(err) { - suite.Equal(errors.New("language: tag is not well-formed"), err) + if suite.NoError(err) { + suite.Equal(nil, err) } err = validate.Language(dutch) @@ -214,6 +217,21 @@ func (suite *ValidationTestSuite) TestValidateLanguage() { if suite.NoError(err) { suite.Equal(nil, err) } + + err = validate.Language(chinese) + if suite.NoError(err) { + suite.Equal(nil, err) + } + + err = validate.Language(chineseSimplified) + if suite.NoError(err) { + suite.Equal(nil, err) + } + + err = validate.Language(chineseTraditional) + if suite.NoError(err) { + suite.Equal(nil, err) + } } func (suite *ValidationTestSuite) TestValidateReason() { diff --git a/internal/validate/normalize/language.go b/internal/validate/normalize/language.go new file mode 100644 index 0000000000..c0c9b4129d --- /dev/null +++ b/internal/validate/normalize/language.go @@ -0,0 +1,34 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package normalize + +import ( + "golang.org/x/text/language" +) + +// Language converts a previously validated but possibly non-canonical BCP 47 tag to its canonical form. +// See: https://pkg.go.dev/golang.org/x/text/language +// See: [internal/validate.Language] +func Language(lang string) string { + canonical, err := language.Parse(lang) + if err != nil { + // Should not happen: input should have been previously validated. + return "" + } + return canonical.String() +} diff --git a/internal/validate/normalize/language_test.go b/internal/validate/normalize/language_test.go new file mode 100644 index 0000000000..fbe64ab4c4 --- /dev/null +++ b/internal/validate/normalize/language_test.go @@ -0,0 +1,89 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package normalize + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type NormalizationTestSuite struct { + suite.Suite +} + +func (suite *NormalizationTestSuite) TestNormalizeLanguage() { + empty := "" + notALanguage := "this isn't a language at all!" + english := "en" + // Should be all lowercase + capitalEnglish := "EN" + // Overlong, should be in ISO 639-1 format + arabic3Letters := "ara" + // Should be all lowercase + mixedCapsEnglish := "eN" + // Region should be capitalized + englishUS := "en-us" + dutch := "nl" + german := "de" + chinese := "zh" + chineseSimplified := "zh-Hans" + chineseTraditional := "zh-Hant" + + var actual string + + actual = Language(empty) + suite.Equal(empty, actual) + + actual = Language(notALanguage) + suite.Equal(empty, actual) + + actual = Language(english) + suite.Equal(english, actual) + + actual = Language(capitalEnglish) + suite.Equal(english, actual) + + actual = Language(arabic3Letters) + suite.Equal("ar", actual) + + actual = Language(mixedCapsEnglish) + suite.Equal(english, actual) + + actual = Language(englishUS) + suite.Equal("en-US", actual) + + actual = Language(dutch) + suite.Equal(dutch, actual) + + actual = Language(german) + suite.Equal(german, actual) + + actual = Language(chinese) + suite.Equal(chinese, actual) + + actual = Language(chineseSimplified) + suite.Equal(chineseSimplified, actual) + + actual = Language(chineseTraditional) + suite.Equal(chineseTraditional, actual) +} + +func TestNormalizationTestSuite(t *testing.T) { + suite.Run(t, new(NormalizationTestSuite)) +} From 4937b7b51767527511b0570180a77a743d85be47 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 6 Aug 2023 11:57:32 -0700 Subject: [PATCH 2/4] Fuse validation and normalization for languages --- internal/api/client/accounts/accountcreate.go | 11 +- internal/api/client/statuses/statuscreate.go | 10 +- internal/processing/account/create.go | 6 +- internal/processing/account/update.go | 6 +- internal/processing/status/create.go | 5 +- internal/validate/formvalidation.go | 13 ++- internal/validate/formvalidation_test.go | 105 ++++++------------ internal/validate/normalize/language.go | 34 ------ internal/validate/normalize/language_test.go | 89 --------------- 9 files changed, 63 insertions(+), 216 deletions(-) delete mode 100644 internal/validate/normalize/language.go delete mode 100644 internal/validate/normalize/language_test.go diff --git a/internal/api/client/accounts/accountcreate.go b/internal/api/client/accounts/accountcreate.go index c8247ecf2a..473000f6d4 100644 --- a/internal/api/client/accounts/accountcreate.go +++ b/internal/api/client/accounts/accountcreate.go @@ -87,7 +87,7 @@ func (m *Module) AccountCreatePOSTHandler(c *gin.Context) { return } - if err := validateCreateAccount(form); err != nil { + if err := validateNormalizeCreateAccount(form); err != nil { apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1) return } @@ -110,9 +110,10 @@ func (m *Module) AccountCreatePOSTHandler(c *gin.Context) { c.JSON(http.StatusOK, ti) } -// validateCreateAccount checks through all the necessary prerequisites for creating a new account, +// validateNormalizeCreateAccount checks through all the necessary prerequisites for creating a new account, // according to the provided account create request. If the account isn't eligible, an error will be returned. -func validateCreateAccount(form *apimodel.AccountCreateRequest) error { +// Side effect: normalizes the provided language tag for the user's locale. +func validateNormalizeCreateAccount(form *apimodel.AccountCreateRequest) error { if form == nil { return errors.New("form was nil") } @@ -137,9 +138,11 @@ func validateCreateAccount(form *apimodel.AccountCreateRequest) error { return errors.New("agreement to terms and conditions not given") } - if err := validate.Language(form.Locale); err != nil { + locale, err := validate.Language(form.Locale) + if err != nil { return err } + form.Locale = locale return validate.SignUpReason(form.Reason, config.GetAccountsReasonRequired()) } diff --git a/internal/api/client/statuses/statuscreate.go b/internal/api/client/statuses/statuscreate.go index e4d8588c72..e8378f4619 100644 --- a/internal/api/client/statuses/statuscreate.go +++ b/internal/api/client/statuses/statuscreate.go @@ -98,7 +98,7 @@ func (m *Module) StatusCreatePOSTHandler(c *gin.Context) { // } // form.Status += "\n\nsent from " + user + "'s iphone\n" - if err := validateCreateStatus(form); err != nil { + if err := validateNormalizeCreateStatus(form); err != nil { apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1) return } @@ -112,7 +112,9 @@ func (m *Module) StatusCreatePOSTHandler(c *gin.Context) { c.JSON(http.StatusOK, apiStatus) } -func validateCreateStatus(form *apimodel.AdvancedStatusCreateForm) error { +// validateNormalizeCreateStatus checks the form for disallowed combinations of attachments and overlength inputs. +// Side effect: normalizes the post's language tag. +func validateNormalizeCreateStatus(form *apimodel.AdvancedStatusCreateForm) error { hasStatus := form.Status != "" hasMedia := len(form.MediaIDs) != 0 hasPoll := form.Poll != nil @@ -162,9 +164,11 @@ func validateCreateStatus(form *apimodel.AdvancedStatusCreateForm) error { } if form.Language != "" { - if err := validate.Language(form.Language); err != nil { + language, err := validate.Language(form.Language) + if err != nil { return err } + form.Language = language } return nil diff --git a/internal/processing/account/create.go b/internal/processing/account/create.go index 98deddd687..32a59d1ef3 100644 --- a/internal/processing/account/create.go +++ b/internal/processing/account/create.go @@ -28,15 +28,13 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" - "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" "github.com/superseriousbusiness/oauth2/v4" ) // Create processes the given form for creating a new account, // returning an oauth token for that account if successful. // -// Fields on the form should have already been validated by the -// caller, before this function is called. +// Precondition: the form's fields should have already been validated and normalized by the caller. func (p *Processor) Create( ctx context.Context, appToken oauth2.TokenInfo, @@ -76,7 +74,7 @@ func (p *Processor) Create( Reason: text.SanitizePlaintext(reason), PreApproved: !config.GetAccountsApprovalRequired(), // Mark as approved if no approval required. SignUpIP: form.IP, - Locale: normalize.Language(form.Locale), + Locale: form.Locale, AppID: app.ID, }) if err != nil { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 3fa058ec36..f75b3c8d97 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -34,7 +34,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/validate" - "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" ) func (p *Processor) selectNoteFormatter(contentType string) text.FormatFunc { @@ -223,10 +222,11 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form if form.Source != nil { if form.Source.Language != nil { - if err := validate.Language(*form.Source.Language); err != nil { + language, err := validate.Language(*form.Source.Language) + if err != nil { return nil, gtserror.NewErrorBadRequest(err) } - account.Language = normalize.Language(*form.Source.Language) + account.Language = language } if form.Source.Sensitive != nil { diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index 68ef76cf2d..36842ee07a 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -34,10 +34,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/uris" - "github.com/superseriousbusiness/gotosocial/internal/validate/normalize" ) // Create processes the given form to create a new status, returning the api model representation of that status if it's OK. +// +// Precondition: the form's fields should have already been validated and normalized by the caller. func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, application *gtsmodel.Application, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, gtserror.WithCode) { accountURIs := uris.GenerateURIsForAccount(account.Username) thisStatusID := id.NewULID() @@ -266,7 +267,7 @@ func processVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateF func processLanguage(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error { if form.Language != "" { - status.Language = normalize.Language(form.Language) + status.Language = form.Language } else { status.Language = accountDefaultLanguage } diff --git a/internal/validate/formvalidation.go b/internal/validate/formvalidation.go index 3bb50a543c..7c5c229a40 100644 --- a/internal/validate/formvalidation.go +++ b/internal/validate/formvalidation.go @@ -100,16 +100,19 @@ func Email(email string) error { } // Language checks that the given language string is a valid, if not necessarily canonical, BCP 47 language tag. +// Returns a canonicalized version of the tag if the language can be parsed. // Returns an error if the language cannot be parsed. -// Does not return an error if the language is not in the canonical tag format. // See: https://pkg.go.dev/golang.org/x/text/language // See: [internal/validate/normalize.Language] -func Language(lang string) error { +func Language(lang string) (string, error) { if lang == "" { - return errors.New("no language provided") + return "", errors.New("no language provided") } - _, err := language.Parse(lang) - return err + parsed, err := language.Parse(lang) + if err != nil { + return "", err + } + return parsed.String(), err } // SignUpReason checks that a sufficient reason is given for a server signup request diff --git a/internal/validate/formvalidation_test.go b/internal/validate/formvalidation_test.go index 32e6c30273..40830407c4 100644 --- a/internal/validate/formvalidation_test.go +++ b/internal/validate/formvalidation_test.go @@ -159,78 +159,39 @@ func (suite *ValidationTestSuite) TestValidateEmail() { } func (suite *ValidationTestSuite) TestValidateLanguage() { - empty := "" - notALanguage := "this isn't a language at all!" - english := "en" - capitalEnglish := "EN" - arabic3Letters := "ara" - mixedCapsEnglish := "eN" - englishUS := "en-us" - dutch := "nl" - german := "de" - chinese := "zh" - chineseSimplified := "zh-Hans" - chineseTraditional := "zh-Hant" - var err error - - err = validate.Language(empty) - if suite.Error(err) { - suite.Equal(errors.New("no language provided"), err) - } - - err = validate.Language(notALanguage) - if suite.Error(err) { - suite.Equal(errors.New("language: tag is not well-formed"), err) - } - - err = validate.Language(english) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(capitalEnglish) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(arabic3Letters) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(mixedCapsEnglish) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(englishUS) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(dutch) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(german) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(chinese) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(chineseSimplified) - if suite.NoError(err) { - suite.Equal(nil, err) - } - - err = validate.Language(chineseTraditional) - if suite.NoError(err) { - suite.Equal(nil, err) + testCases := []struct { + name, input, expected, err string + }{ + {name: "empty", err: "no language provided"}, + {name: "notALanguage", input: "this isn't a language at all!", err: "language: tag is not well-formed"}, + {name: "english", input: "en", expected: "en"}, + // Should be all lowercase + {name: "capitalEnglish", input: "EN", expected: "en"}, + // Overlong, should be in ISO 639-1 format + {name: "arabic3Letters", input: "ara", expected: "ar"}, + // Should be all lowercase + {name: "mixedCapsEnglish", input: "eN", expected: "en"}, + // Region should be capitalized + {name: "englishUS", input: "en-us", expected: "en-US"}, + {name: "dutch", input: "nl", expected: "nl"}, + {name: "german", input: "de", expected: "de"}, + {name: "chinese", input: "zh", expected: "zh"}, + {name: "chineseSimplified", input: "zh-Hans", expected: "zh-Hans"}, + {name: "chineseTraditional", input: "zh-Hant", expected: "zh-Hant"}, + } + + for _, testCase := range testCases { + testCase := testCase + suite.Run(testCase.name, func() { + actual, actualErr := validate.Language(testCase.input) + if testCase.err == "" { + suite.Equal(testCase.expected, actual) + suite.NoError(actualErr) + } else { + suite.Empty(actual) + suite.EqualError(actualErr, testCase.err) + } + }) } } diff --git a/internal/validate/normalize/language.go b/internal/validate/normalize/language.go deleted file mode 100644 index c0c9b4129d..0000000000 --- a/internal/validate/normalize/language.go +++ /dev/null @@ -1,34 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package normalize - -import ( - "golang.org/x/text/language" -) - -// Language converts a previously validated but possibly non-canonical BCP 47 tag to its canonical form. -// See: https://pkg.go.dev/golang.org/x/text/language -// See: [internal/validate.Language] -func Language(lang string) string { - canonical, err := language.Parse(lang) - if err != nil { - // Should not happen: input should have been previously validated. - return "" - } - return canonical.String() -} diff --git a/internal/validate/normalize/language_test.go b/internal/validate/normalize/language_test.go deleted file mode 100644 index fbe64ab4c4..0000000000 --- a/internal/validate/normalize/language_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package normalize - -import ( - "testing" - - "github.com/stretchr/testify/suite" -) - -type NormalizationTestSuite struct { - suite.Suite -} - -func (suite *NormalizationTestSuite) TestNormalizeLanguage() { - empty := "" - notALanguage := "this isn't a language at all!" - english := "en" - // Should be all lowercase - capitalEnglish := "EN" - // Overlong, should be in ISO 639-1 format - arabic3Letters := "ara" - // Should be all lowercase - mixedCapsEnglish := "eN" - // Region should be capitalized - englishUS := "en-us" - dutch := "nl" - german := "de" - chinese := "zh" - chineseSimplified := "zh-Hans" - chineseTraditional := "zh-Hant" - - var actual string - - actual = Language(empty) - suite.Equal(empty, actual) - - actual = Language(notALanguage) - suite.Equal(empty, actual) - - actual = Language(english) - suite.Equal(english, actual) - - actual = Language(capitalEnglish) - suite.Equal(english, actual) - - actual = Language(arabic3Letters) - suite.Equal("ar", actual) - - actual = Language(mixedCapsEnglish) - suite.Equal(english, actual) - - actual = Language(englishUS) - suite.Equal("en-US", actual) - - actual = Language(dutch) - suite.Equal(dutch, actual) - - actual = Language(german) - suite.Equal(german, actual) - - actual = Language(chinese) - suite.Equal(chinese, actual) - - actual = Language(chineseSimplified) - suite.Equal(chineseSimplified, actual) - - actual = Language(chineseTraditional) - suite.Equal(chineseTraditional, actual) -} - -func TestNormalizationTestSuite(t *testing.T) { - suite.Run(t, new(NormalizationTestSuite)) -} From 6b899a622e36c63390b7216dd30842396137bf7d Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 6 Aug 2023 12:00:22 -0700 Subject: [PATCH 3/4] Remove outdated comment line --- internal/validate/formvalidation.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/validate/formvalidation.go b/internal/validate/formvalidation.go index 7c5c229a40..1b5996b4b0 100644 --- a/internal/validate/formvalidation.go +++ b/internal/validate/formvalidation.go @@ -103,7 +103,6 @@ func Email(email string) error { // Returns a canonicalized version of the tag if the language can be parsed. // Returns an error if the language cannot be parsed. // See: https://pkg.go.dev/golang.org/x/text/language -// See: [internal/validate/normalize.Language] func Language(lang string) (string, error) { if lang == "" { return "", errors.New("no language provided") From 7e757ec060b259d16b33681a9435873a33bde357 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 6 Aug 2023 14:28:15 -0700 Subject: [PATCH 4/4] Move post language canonicalization test --- .../api/client/statuses/statuscreate_test.go | 36 +++++++++++++++++++ internal/processing/status/create_test.go | 34 ------------------ 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go index 05f24c24c0..d47a74bbc2 100644 --- a/internal/api/client/statuses/statuscreate_test.go +++ b/internal/api/client/statuses/statuscreate_test.go @@ -391,6 +391,42 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() { suite.Equal(statusResponse.ID, gtsAttachment.StatusID) } +// Post a new status with a language tag that is not in canonical format +func (suite *StatusCreateTestSuite) TestPostNewStatusWithNoncanonicalLanguageTag() { + t := suite.testTokens["local_account_1"] + oauthToken := oauth.DBTokenToToken(t) + + // setup + recorder := httptest.NewRecorder() + ctx, _ := testrig.CreateGinTestContext(recorder, nil) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedToken, oauthToken) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) + ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) + ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", statuses.BasePath), nil) // the endpoint we're hitting + ctx.Request.Header.Set("accept", "application/json") + ctx.Request.Form = url.Values{ + "status": {"English? what's English? i speak American"}, + "language": {"en-us"}, + } + suite.statusModule.StatusCreatePOSTHandler(ctx) + + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + suite.NoError(err) + + statusReply := &apimodel.Status{} + err = json.Unmarshal(b, statusReply) + suite.NoError(err) + + suite.Equal("

English? what's English? i speak American

", statusReply.Content) + suite.NotNil(statusReply.Language) + suite.Equal("en-US", *statusReply.Language) +} + func TestStatusCreateTestSuite(t *testing.T) { suite.Run(t, new(StatusCreateTestSuite)) } diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go index 701dbf3e80..2c86e5a29d 100644 --- a/internal/processing/status/create_test.go +++ b/internal/processing/status/create_test.go @@ -242,40 +242,6 @@ func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() { suite.Equal("zh-Hans", *apiStatus.Language) } -func (suite *StatusCreateTestSuite) TestProcessLanguageWithNoncanonicalLanguageTag() { - ctx := context.Background() - - creatingAccount := suite.testAccounts["local_account_1"] - creatingApplication := suite.testApplications["application_1"] - - statusCreateForm := &apimodel.AdvancedStatusCreateForm{ - StatusCreateRequest: apimodel.StatusCreateRequest{ - Status: "hello world", - MediaIDs: []string{}, - Poll: nil, - InReplyToID: "", - Sensitive: false, - SpoilerText: "", - Visibility: apimodel.VisibilityPublic, - ScheduledAt: "", - Language: "en-us", - ContentType: apimodel.StatusContentTypePlain, - }, - AdvancedVisibilityFlagsForm: apimodel.AdvancedVisibilityFlagsForm{ - Federated: nil, - Boostable: nil, - Replyable: nil, - Likeable: nil, - }, - } - - apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm) - suite.NoError(err) - suite.NotNil(apiStatus) - - suite.Equal("en-US", *apiStatus.Language) -} - func TestStatusCreateTestSuite(t *testing.T) { suite.Run(t, new(StatusCreateTestSuite)) }