Skip to content

Commit

Permalink
Git LFS support v2 (#122)
Browse files Browse the repository at this point in the history
* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: git-lfs/git-lfs#1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled
  • Loading branch information
fabian-z authored and lunny committed Dec 26, 2016
1 parent 4b7594d commit 2e7ccec
Show file tree
Hide file tree
Showing 37 changed files with 2,632 additions and 11 deletions.
64 changes: 63 additions & 1 deletion cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package cmd

import (
"crypto/tls"
"encoding/json"
"fmt"
"os"
"os/exec"
Expand All @@ -21,12 +22,14 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/Unknwon/com"
"github.com/dgrijalva/jwt-go"
gouuid "github.com/satori/go.uuid"
"github.com/urfave/cli"
)

const (
accessDenied = "Repository does not exist or you do not have access"
accessDenied = "Repository does not exist or you do not have access"
lfsAuthenticateVerb = "git-lfs-authenticate"
)

// CmdServ represents the available serv sub-command.
Expand Down Expand Up @@ -73,6 +76,7 @@ var (
"git-upload-pack": models.AccessModeRead,
"git-upload-archive": models.AccessModeRead,
"git-receive-pack": models.AccessModeWrite,
lfsAuthenticateVerb: models.AccessModeNone,
}
)

Expand Down Expand Up @@ -161,6 +165,21 @@ func runServ(c *cli.Context) error {
}

verb, args := parseCmd(cmd)

var lfsVerb string
if verb == lfsAuthenticateVerb {

if !setting.LFS.StartServer {
fail("Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
}

if strings.Contains(args, " ") {
argsSplit := strings.SplitN(args, " ", 2)
args = strings.TrimSpace(argsSplit[0])
lfsVerb = strings.TrimSpace(argsSplit[1])
}
}

repoPath := strings.ToLower(strings.Trim(args, "'"))
rr := strings.SplitN(repoPath, "/", 2)
if len(rr) != 2 {
Expand Down Expand Up @@ -196,6 +215,14 @@ func runServ(c *cli.Context) error {
fail("Unknown git command", "Unknown git command %s", verb)
}

if verb == lfsAuthenticateVerb {
if lfsVerb == "upload" {
requestedMode = models.AccessModeWrite
} else {
requestedMode = models.AccessModeRead
}
}

// Prohibit push to mirror repositories.
if requestedMode > models.AccessModeRead && repo.IsMirror {
fail("mirror repository is read-only", "")
Expand Down Expand Up @@ -261,6 +288,41 @@ func runServ(c *cli.Context) error {
}
}

//LFS token authentication

if verb == lfsAuthenticateVerb {

url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, repoUser.Name, repo.Name)

now := time.Now()
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"repo": repo.ID,
"op": lfsVerb,
"exp": now.Add(5 * time.Minute).Unix(),
"nbf": now.Unix(),
})

// Sign and get the complete encoded token as a string using the secret
tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
if err != nil {
fail("Internal error", "Failed to sign JWT token: %v", err)
}

tokenAuthentication := &models.LFSTokenResponse{
Header: make(map[string]string),
Href: url,
}
tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString)

enc := json.NewEncoder(os.Stdout)
err = enc.Encode(tokenAuthentication)
if err != nil {
fail("Internal error", "Failed to encode LFS json response: %v", err)
}

return nil
}

uuid := gouuid.NewV4().String()
os.Setenv("GITEA_UUID", uuid)
// Keep the old env variable name for backward compability
Expand Down
12 changes: 12 additions & 0 deletions cmd/web.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/public"
Expand All @@ -29,6 +30,7 @@ import (
"code.gitea.io/gitea/routers/org"
"code.gitea.io/gitea/routers/repo"
"code.gitea.io/gitea/routers/user"

"github.com/go-macaron/binding"
"github.com/go-macaron/cache"
"github.com/go-macaron/captcha"
Expand Down Expand Up @@ -564,6 +566,12 @@ func runWeb(ctx *cli.Context) error {
}, ignSignIn, context.RepoAssignment(true), context.RepoRef())

m.Group("/:reponame", func() {
m.Group("/info/lfs", func() {
m.Post("/objects/batch", lfs.BatchHandler)
m.Get("/objects/:oid/:filename", lfs.ObjectOidHandler)
m.Any("/objects/:oid", lfs.ObjectOidHandler)
m.Post("/objects", lfs.PostHandler)
}, ignSignInAndCsrf)
m.Any("/*", ignSignInAndCsrf, repo.HTTP)
m.Head("/tasks/trigger", repo.TriggerTask)
})
Expand Down Expand Up @@ -600,6 +608,10 @@ func runWeb(ctx *cli.Context) error {
}
log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)

if setting.LFS.StartServer {
log.Info("LFS server enabled")
}

var err error
switch setting.Protocol {
case setting.HTTP:
Expand Down
25 changes: 25 additions & 0 deletions models/git_diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ type DiffFile struct {
IsCreated bool
IsDeleted bool
IsBin bool
IsLFSFile bool
IsRenamed bool
IsSubmodule bool
Sections []*DiffSection
Expand Down Expand Up @@ -245,6 +246,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
leftLine, rightLine int
lineCount int
curFileLinesCount int
curFileLFSPrefix bool
)

input := bufio.NewReader(reader)
Expand All @@ -268,6 +270,28 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
continue
}

trimLine := strings.Trim(line, "+- ")

if trimLine == LFSMetaFileIdentifier {
curFileLFSPrefix = true
}

if curFileLFSPrefix && strings.HasPrefix(trimLine, LFSMetaFileOidPrefix) {
oid := strings.TrimPrefix(trimLine, LFSMetaFileOidPrefix)

if len(oid) == 64 {
m := &LFSMetaObject{Oid: oid}
count, err := x.Count(m)

if err == nil && count > 0 {
curFile.IsBin = true
curFile.IsLFSFile = true
curSection.Lines = nil
break
}
}
}

curFileLinesCount++
lineCount++

Expand Down Expand Up @@ -354,6 +378,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
break
}
curFileLinesCount = 0
curFileLFSPrefix = false

// Check file diff type and is submodule.
for {
Expand Down
122 changes: 122 additions & 0 deletions models/lfs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package models

import (
"errors"
"github.com/go-xorm/xorm"
"time"
)

// LFSMetaObject stores metadata for LFS tracked files.
type LFSMetaObject struct {
ID int64 `xorm:"pk autoincr"`
Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"`
Size int64 `xorm:"NOT NULL"`
RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
Existing bool `xorm:"-"`
Created time.Time `xorm:"-"`
CreatedUnix int64
}

// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
// This structure is fetched via SSH and passed by the Git LFS client to the server
// endpoint for authorization.
type LFSTokenResponse struct {
Header map[string]string `json:"header"`
Href string `json:"href"`
}

var (
// ErrLFSObjectNotExist is returned from lfs models functions in order
// to differentiate between database and missing object errors.
ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist")
)

const (
// LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files.
// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"

// LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
LFSMetaFileOidPrefix = "oid sha256:"
)

// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
// if it is not already present.
func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
var err error

has, err := x.Get(m)
if err != nil {
return nil, err
}

if has {
m.Existing = true
return m, nil
}

sess := x.NewSession()
defer sessionRelease(sess)
if err = sess.Begin(); err != nil {
return nil, err
}

if _, err = sess.Insert(m); err != nil {
return nil, err
}

return m, sess.Commit()
}

// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
// the returned pointer is a valid LFSMetaObject.
func GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error) {
if len(oid) == 0 {
return nil, ErrLFSObjectNotExist
}

m := &LFSMetaObject{Oid: oid}
has, err := x.Get(m)
if err != nil {
return nil, err
} else if !has {
return nil, ErrLFSObjectNotExist
}
return m, nil
}

// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error.
func RemoveLFSMetaObjectByOid(oid string) error {
if len(oid) == 0 {
return ErrLFSObjectNotExist
}

sess := x.NewSession()
defer sessionRelease(sess)
if err := sess.Begin(); err != nil {
return err
}

m := &LFSMetaObject{Oid: oid}

if _, err := sess.Delete(m); err != nil {
return err
}

return sess.Commit()
}

// BeforeInsert sets the time at which the LFSMetaObject was created.
func (m *LFSMetaObject) BeforeInsert() {
m.CreatedUnix = time.Now().Unix()
}

// AfterSet stores the LFSMetaObject creation time in the database as local time.
func (m *LFSMetaObject) AfterSet(colName string, _ xorm.Cell) {
switch colName {
case "created_unix":
m.Created = time.Unix(m.CreatedUnix, 0).Local()
}
}
2 changes: 1 addition & 1 deletion models/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func init() {
new(Mirror), new(Release), new(LoginSource), new(Webhook),
new(UpdateTask), new(HookTask),
new(Team), new(OrgUser), new(TeamUser), new(TeamRepo),
new(Notice), new(EmailAddress))
new(Notice), new(EmailAddress), new(LFSMetaObject))

gonicNames := []string{"SSL", "UID"}
for _, name := range gonicNames {
Expand Down
Loading

0 comments on commit 2e7ccec

Please sign in to comment.