add redis task queue support and improve docs

.gitignore

@@ -73,3 +73,4 @@ prime/
 *.snap
 *.snap-build
 *_source.tar.bz2
+.DS_Store

custom/conf/app.ini.sample

@@ -716,3 +716,8 @@ IS_INPUT_FILE = false
 ENABLED = false
 ; If you want to add authorization, specify a token here
 TOKEN =
+
+[task]
+QUEUE_TYPE = redis
+QUEUE_LENGTH = 1000
+QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -401,8 +401,15 @@ Two special environment variables are passed to the render command:
 - `GITEA_PREFIX_SRC`, which contains the current URL prefix in the `src` path tree. To be used as prefix for links.
 - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths.
 
+## Task (`task`)
+
+- `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`.
+- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
+- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connction string, available only when `QUEUE_TYPE` is `redis`. If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
 - `SHOW_FOOTER_VERSION`: **true**: Show Gitea version information in the footer.
 - `SHOW_FOOTER_TEMPLATE_LOAD_TIME`: **true**: Show time of template execution in the footer.
+

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -234,7 +234,11 @@ IS_INPUT_FILE = false
 - RENDER_COMMAND: 工具的命令行命令及参数。
 - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。
 
+## Task (`task`)
 
+- `QUEUE_TYPE`: **channel**: 任务队列类型，可以为 `channel` 或 `redis`。
+- `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。
+- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。
 
 ## Other (`other`)
 

modules/setting/indexer.go

@@ -13,6 +13,7 @@ import (
 const (
 	LevelQueueType   = "levelqueue"
 	ChannelQueueType = "channel"
+	RedisQueueType   = "redis"
 )
 
 var (

modules/setting/task.go

@@ -7,16 +7,19 @@ package setting
 var (
 	// Task settings
 	Task = struct {
-		QueueType   string
-		QueueLength int
+		QueueType    string
+		QueueLength  int
+		QueueConnStr string
 	}{
-		QueueType:   ChannelQueueType,
-		QueueLength: 1000,
+		QueueType:    ChannelQueueType,
+		QueueLength:  1000,
+		QueueConnStr: "addrs=127.0.0.1:6379 db=0",
 	}
 )
 
 func newTaskService() {
 	sec := Cfg.Section("task")
 	Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType)
 	Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
+	Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0")
 }

modules/task/queue_redis.go

@@ -0,0 +1,119 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/go-redis/redis"
+)
+
+var (
+	_ Queue = &RedisQueue{}
+)
+
+type redisClient interface {
+	RPush(key string, args ...interface{}) *redis.IntCmd
+	LPop(key string) *redis.StringCmd
+	Ping() *redis.StatusCmd
+}
+
+// RedisQueue redis queue
+type RedisQueue struct {
+	client    redisClient
+	queueName string
+}
+
+func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) {
+	fields := strings.Fields(connStr)
+	for _, f := range fields {
+		items := strings.SplitN(f, "=", 2)
+		if len(items) < 2 {
+			continue
+		}
+		switch strings.ToLower(items[0]) {
+		case "addrs":
+			addrs = items[1]
+		case "password":
+			password = items[1]
+		case "db":
+			dbIdx, err = strconv.Atoi(items[1])
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// NewRedisQueue creates single redis or cluster redis queue
+func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) {
+	dbs := strings.Split(addrs, ",")
+	var queue = RedisQueue{
+		queueName: "task_queue",
+	}
+	if len(dbs) == 0 {
+		return nil, errors.New("no redis host found")
+	} else if len(dbs) == 1 {
+		queue.client = redis.NewClient(&redis.Options{
+			Addr:     strings.TrimSpace(dbs[0]), // use default Addr
+			Password: password,                  // no password set
+			DB:       dbIdx,                     // use default DB
+		})
+	} else {
+		// cluster will ignore db
+		queue.client = redis.NewClusterClient(&redis.ClusterOptions{
+			Addrs:    dbs,
+			Password: password,
+		})
+	}
+	if err := queue.client.Ping().Err(); err != nil {
+		return nil, err
+	}
+	return &queue, nil
+}
+
+func (r *RedisQueue) Run() error {
+	for {
+		bs, err := r.client.LPop(r.queueName).Bytes()
+		if err != nil {
+			if err != redis.Nil {
+				log.Error(4, "LPop failed: %v", err)
+			}
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		var task models.Task
+		err = json.Unmarshal(bs, &task)
+		if err != nil {
+			log.Error(4, "Unmarshal task failed: %s", err.Error())
+		} else {
+			err = Run(&task)
+			if err != nil {
+				log.Error(4, "Run task failed: %s", err.Error())
+			}
+		}
+
+		time.Sleep(time.Millisecond * 100)
+	}
+	return nil
+}
+
+// Push implements Queue
+func (r *RedisQueue) Push(task *models.Task) error {
+	bs, err := json.Marshal(task)
+	if err != nil {
+		return err
+	}
+	return r.client.RPush(r.queueName, bs).Err()
+}

modules/task/task.go

@@ -31,8 +31,17 @@ func Init() error {
 	switch setting.Task.QueueType {
 	case setting.ChannelQueueType:
 		taskQueue = NewChannelQueue(setting.Task.QueueLength)
+	case setting.RedisQueueType:
+		addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr)
+		if err != nil {
+			return err
+		}
+		taskQueue, err = NewRedisQueue(addrs, pass, idx)
+		if err != nil {
+			return err
+		}
 	default:
-		return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueIndexerQueueType)
+		return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType)
 	}
 
 	go taskQueue.Run()

public/js/index.js

@@ -229,15 +229,14 @@ function updateIssuesMeta(url, action, issueIds, elementId) {
             },
             success: resolve
         })
-    }
+    })
 }
 
 function initRepoStatusChecker() {
-    console.log("initRepoStatusChecker")
     var migrating = $("#repo_migrating");
     if (migrating) {
         var repo_name = migrating.attr('repo');
-        if (typeof repo_nane === 'undefined') {
+        if (typeof repo_name === 'undefined') {
             return
         }
         $.ajax({

templates/repo/migrating.tmpl

@@ -7,7 +7,7 @@
 				{{template "base/alert" .}}
 				<div class="home">
 					<div class="ui stackable middle very relaxed page grid">
-						<div id="reop_migrating" class="sixteen wide center aligned centered column" repo="{{.Repo.Repository.FullName}}">
+						<div id="repo_migrating" class="sixteen wide center aligned centered column" repo="{{.Repo.Repository.FullName}}">
 							<div>
 								<img src="{{AppSubUrl}}/img/loading.png"/>
 							</div>