diff --git a/.gitignore b/.gitignore index 2fe0134f7d884..aad17b296d95b 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ prime/ *.snap *.snap-build *_source.tar.bz2 +.DS_Store \ No newline at end of file diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9323e5a0d5867..ce3075f3dca10 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -716,3 +716,8 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[task] +QUEUE_TYPE = redis +QUEUE_LENGTH = 1000 +QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index a7e8f767367fa..af492bd593125 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -401,8 +401,15 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_SRC`, which contains the current URL prefix in the `src` path tree. To be used as prefix for links. - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`. +- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`. +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connction string, available only when `QUEUE_TYPE` is `redis`. If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. - `SHOW_FOOTER_VERSION`: **true**: Show Gitea version information in the footer. - `SHOW_FOOTER_TEMPLATE_LOAD_TIME`: **true**: Show time of template execution in the footer. + diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index b1b2f11f298c0..ab30420242ba7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -234,7 +234,11 @@ IS_INPUT_FILE = false - RENDER_COMMAND: 工具的命令行命令及参数。 - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 +## Task (`task`) +- `QUEUE_TYPE`: **channel**: 任务队列类型，可以为 `channel` 或 `redis`。 +- `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。 +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。 ## Other (`other`) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 0e09ed2b86a31..79be3bfd7c6fb 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -13,6 +13,7 @@ import ( const ( LevelQueueType = "levelqueue" ChannelQueueType = "channel" + RedisQueueType = "redis" ) var ( diff --git a/modules/setting/task.go b/modules/setting/task.go index adcb24fa67212..97704d4a4da68 100644 --- a/modules/setting/task.go +++ b/modules/setting/task.go @@ -7,11 +7,13 @@ package setting var ( // Task settings Task = struct { - QueueType string - QueueLength int + QueueType string + QueueLength int + QueueConnStr string }{ - QueueType: ChannelQueueType, - QueueLength: 1000, + QueueType: ChannelQueueType, + QueueLength: 1000, + QueueConnStr: "addrs=127.0.0.1:6379 db=0", } ) @@ -19,4 +21,5 @@ func newTaskService() { sec := Cfg.Section("task") Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) + Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") } diff --git a/modules/task/queue_redis.go b/modules/task/queue_redis.go new file mode 100644 index 0000000000000..7439bfd6270d0 --- /dev/null +++ b/modules/task/queue_redis.go @@ -0,0 +1,119 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "github.com/go-redis/redis" +) + +var ( + _ Queue = &RedisQueue{} +) + +type redisClient interface { + RPush(key string, args ...interface{}) *redis.IntCmd + LPop(key string) *redis.StringCmd + Ping() *redis.StatusCmd +} + +// RedisQueue redis queue +type RedisQueue struct { + client redisClient + queueName string +} + +func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { + fields := strings.Fields(connStr) + for _, f := range fields { + items := strings.SplitN(f, "=", 2) + if len(items) < 2 { + continue + } + switch strings.ToLower(items[0]) { + case "addrs": + addrs = items[1] + case "password": + password = items[1] + case "db": + dbIdx, err = strconv.Atoi(items[1]) + if err != nil { + return + } + } + } + return +} + +// NewRedisQueue creates single redis or cluster redis queue +func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { + dbs := strings.Split(addrs, ",") + var queue = RedisQueue{ + queueName: "task_queue", + } + if len(dbs) == 0 { + return nil, errors.New("no redis host found") + } else if len(dbs) == 1 { + queue.client = redis.NewClient(&redis.Options{ + Addr: strings.TrimSpace(dbs[0]), // use default Addr + Password: password, // no password set + DB: dbIdx, // use default DB + }) + } else { + // cluster will ignore db + queue.client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: dbs, + Password: password, + }) + } + if err := queue.client.Ping().Err(); err != nil { + return nil, err + } + return &queue, nil +} + +func (r *RedisQueue) Run() error { + for { + bs, err := r.client.LPop(r.queueName).Bytes() + if err != nil { + if err != redis.Nil { + log.Error(4, "LPop failed: %v", err) + } + time.Sleep(time.Millisecond * 100) + continue + } + + var task models.Task + err = json.Unmarshal(bs, &task) + if err != nil { + log.Error(4, "Unmarshal task failed: %s", err.Error()) + } else { + err = Run(&task) + if err != nil { + log.Error(4, "Run task failed: %s", err.Error()) + } + } + + time.Sleep(time.Millisecond * 100) + } + return nil +} + +// Push implements Queue +func (r *RedisQueue) Push(task *models.Task) error { + bs, err := json.Marshal(task) + if err != nil { + return err + } + return r.client.RPush(r.queueName, bs).Err() +} diff --git a/modules/task/task.go b/modules/task/task.go index f072a1f0fce9e..949e5fcef0a35 100644 --- a/modules/task/task.go +++ b/modules/task/task.go @@ -31,8 +31,17 @@ func Init() error { switch setting.Task.QueueType { case setting.ChannelQueueType: taskQueue = NewChannelQueue(setting.Task.QueueLength) + case setting.RedisQueueType: + addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) + if err != nil { + return err + } + taskQueue, err = NewRedisQueue(addrs, pass, idx) + if err != nil { + return err + } default: - return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueIndexerQueueType) + return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) } go taskQueue.Run() diff --git a/public/img/loading.png b/public/img/loading.png index 4f65305d27fce..aac702cfd6d01 100644 Binary files a/public/img/loading.png and b/public/img/loading.png differ diff --git a/public/js/index.js b/public/js/index.js index 24fe8d7ace612..2ba11e150121c 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -229,15 +229,14 @@ function updateIssuesMeta(url, action, issueIds, elementId) { }, success: resolve }) - } + }) } function initRepoStatusChecker() { - console.log("initRepoStatusChecker") var migrating = $("#repo_migrating"); if (migrating) { var repo_name = migrating.attr('repo'); - if (typeof repo_nane === 'undefined') { + if (typeof repo_name === 'undefined') { return } $.ajax({ diff --git a/templates/repo/migrating.tmpl b/templates/repo/migrating.tmpl index 46a95dda1108a..c6b64d78a1325 100644 --- a/templates/repo/migrating.tmpl +++ b/templates/repo/migrating.tmpl @@ -7,7 +7,7 @@ {{template "base/alert" .}}
-
+