Experiment (NOT FOR MERGING): Add test to stress insert ... select for Issue

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e // indirect
 	github.com/go-redis/redis v6.15.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
-	github.com/go-xorm/xorm v0.7.4
+	github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b
 	github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
 	github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14
 	github.com/google/go-github/v24 v24.0.1
@@ -116,6 +116,6 @@ require (
 	gopkg.in/testfixtures.v2 v2.5.0
 	mvdan.cc/xurls/v2 v2.0.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a
-	xorm.io/builder v0.3.5
-	xorm.io/core v0.6.3
+	xorm.io/builder v0.3.6-0.20190711024831-7cc28f8a886b
+	xorm.io/core v0.7.0
 )

go.sum

@@ -143,6 +143,8 @@ github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk
 github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM=
 github.com/go-xorm/xorm v0.7.4 h1:g/NgC590SzqV5VKmdRDNe/K3Holw3YJUCXX28r+rFGw=
 github.com/go-xorm/xorm v0.7.4/go.mod h1:vpza5fydeRgt+stvo9qgMhSNohYqmNt0I1/D6hkCekA=
+github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b h1:Y0hWUheXDHpIs7BWtJcykO4d1VOsVDKg1PsP5YJwxxM=
+github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b/go.mod h1:nqz2TAsuOHWH2yk4FYWtacCGgdbrcdZ5mF1XadqEHls=
 github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561 h1:deE7ritpK04PgtpyVOS2TYcQEld9qLCD5b5EbVNOuLA=
 github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:YgYOrVn3Nj9Tq0EvjmFbphRytDj7JNRoWSStJZWDJTQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -430,6 +432,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -446,6 +449,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190731214159-1e85ed8060aa h1:kwa/4M1dbmhZqOIqYiTtbA6JrvPwo1+jqlub2qDXX90=
@@ -454,6 +458,7 @@ google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMt
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -507,5 +512,9 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a h1:8q33Shx
 strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY=
 xorm.io/builder v0.3.5 h1:EilU39fvWDxjb1cDaELpYhsF+zziRBhew8xk4pngO+A=
 xorm.io/builder v0.3.5/go.mod h1:ZFbByS/KxZI1FKRjL05PyJ4YrK2bcxlUaAxdum5aTR8=
+xorm.io/builder v0.3.6-0.20190711024831-7cc28f8a886b h1:AoM8gTQ/b+ELR104dhRXKb4XO0/NQoBSzZoC2OxwjOg=
+xorm.io/builder v0.3.6-0.20190711024831-7cc28f8a886b/go.mod h1:ZFbByS/KxZI1FKRjL05PyJ4YrK2bcxlUaAxdum5aTR8=
 xorm.io/core v0.6.3 h1:n1NhVZt1s2oLw1BZfX2ocIJsHyso259uPgg63BGr37M=
 xorm.io/core v0.6.3/go.mod h1:8kz/C6arVW/O9vk3PgCiMJO2hIAm1UcuOL3dSPyZ2qo=
+xorm.io/core v0.7.0 h1:hKxuOKWZNeiFQsSuGet/KV8HZ788hclvAl+7azx3tkM=
+xorm.io/core v0.7.0/go.mod h1:TuOJjIVa7e3w/rN8tDcAvuLBMtwzdHPbyOzE6Gk1EUI=

integrations/issue_stress_test.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models"
+)
+
+func TestIssueNoDupIndex(t *testing.T) {
+	prepareTestEnv(t)
+
+	models.TestIssueNoDupIndex(t)
+}

models/issue.go

@@ -1055,12 +1055,6 @@ func getMaxIndexOfIssue(e Engine, repoID int64) (int64, error) {
 func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
 	opts.Issue.Title = strings.TrimSpace(opts.Issue.Title)
 
-	maxIndex, err := getMaxIndexOfIssue(e, opts.Issue.RepoID)
-	if err != nil {
-		return err
-	}
-	opts.Issue.Index = maxIndex + 1
-
 	if opts.Issue.MilestoneID > 0 {
 		milestone, err := getMilestoneByRepoID(e, opts.Issue.RepoID, opts.Issue.MilestoneID)
 		if err != nil && !IsErrMilestoneNotExist(err) {
@@ -1109,10 +1103,20 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
 	}
 
 	// Milestone and assignee validation should happen before insert actual object.
-	if _, err = e.Insert(opts.Issue); err != nil {
+	if _, err := e.SetExpr("`index`", "coalesce(MAX(`index`),0)+1").
+		Where("repo_id=?", opts.Issue.RepoID).
+		Insert(opts.Issue); err != nil {
 		return err
 	}
 
+	inserted, err := getIssueByID(e, opts.Issue.ID)
+	if err != nil {
+		return err
+	}
+
+	// Patch Index with the value calculated by the database
+	opts.Issue.Index = inserted.Index
+
 	if opts.Issue.MilestoneID > 0 {
 		if err = changeMilestoneAssign(e, doer, opts.Issue, -1); err != nil {
 			return err

models/issue_stress.go

@@ -0,0 +1,109 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestIssueNoDupIndex Performs a stress test of the INSERT ... SELECT function of database for inserting issues
+func TestIssueNoDupIndex(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	const initialIssueFill = 1000 // issues inserted prior to stress test
+	const maxTestDuration = 60    // seconds
+	const threadCount = 8         // max simultaneous threads
+	const useTransactions = true  // true: wrap attempts with BEGIN TRANSACTION/COMMIT
+
+	var err error
+	repo := AssertExistsAndLoadBean(t, &Repository{ID: 3}).(*Repository)
+	doer := AssertExistsAndLoadBean(t, &User{ID: repo.OwnerID}).(*User)
+
+	// Pre-load
+	for i := 1; i < initialIssueFill; i++ {
+		issue := &Issue{
+			RepoID:   repo.ID,
+			PosterID: repo.OwnerID,
+			Index:    int64(i + 5000), // Avoid clashing with other tests
+			Title:    fmt.Sprintf("NoDup initial %d", i),
+		}
+		_, err = x.Insert(issue)
+		assert.NoError(t, err)
+	}
+
+	fmt.Printf("TestIssueNoDupIndex(): %d rows created\n", initialIssueFill)
+
+	until := time.Now().Add(time.Second * maxTestDuration)
+
+	var hasErrors int32
+	var wg sync.WaitGroup
+
+	f := func(thread int) {
+		defer wg.Done()
+		sess := x.NewSession()
+		defer sess.Close()
+		i := 1
+		for {
+			if time.Now().After(until) || atomic.LoadInt32(&hasErrors) != 0 {
+				return
+			}
+			issue := &Issue{
+				RepoID:         repo.ID,
+				PosterID:       repo.OwnerID,
+				Title:          fmt.Sprintf("NoDup stress %d, %d", thread, i),
+				OriginalAuthor: "TestIssueNoDupIndex()",
+				Priority:       thread, // For statistics
+			}
+			if useTransactions {
+				if err = sess.Begin(); err != nil {
+					break
+				}
+			}
+			if err = newIssue(sess, doer, NewIssueOptions{
+				Repo:  repo,
+				Issue: issue,
+			}); err != nil {
+				break
+			}
+			if useTransactions {
+				if err = sess.Commit(); err != nil {
+					break
+				}
+			}
+			i++
+		}
+		if useTransactions {
+			_ = sess.Rollback()
+		}
+		atomic.StoreInt32(&hasErrors, 1)
+		t.Logf("newIssue(): %+v", err)
+	}
+
+	for i := 1; i <= threadCount; i++ {
+		go f(i)
+		wg.Add(1)
+	}
+
+	fmt.Printf("TestIssueNoDupIndex(): %d threads created\n", threadCount)
+
+	wg.Wait()
+
+	for i := 1; i <= threadCount; i++ {
+		total, err := x.Table("issue").
+			Where("original_author = ?", "TestIssueNoDupIndex()").
+			And("priority = ?", i).
+			Count()
+		assert.NoError(t, err, "Failed counting generated issues count")
+		fmt.Printf("TestIssueNoDupIndex(): rows created in thread #%d: %d\n", i, total)
+	}
+
+	assert.Equal(t, int32(0), hasErrors, "Synchronization errors detected.")
+}