mirror of
https://github.com/go-gitea/gitea.git
synced 2024-08-20 20:54:21 -04:00
* Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
124 lines
3.2 KiB
Go
124 lines
3.2 KiB
Go
// Copyright 2021 The Gitea Authors. All rights reserved.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package lfs
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
const (
|
|
blobSizeCutoff = 1024
|
|
|
|
// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
|
|
// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
|
|
MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
|
|
|
|
// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
|
|
MetaFileOidPrefix = "oid sha256:"
|
|
)
|
|
|
|
var (
|
|
// ErrMissingPrefix occurs if the content lacks the LFS prefix
|
|
ErrMissingPrefix = errors.New("Content lacks the LFS prefix")
|
|
|
|
// ErrInvalidStructure occurs if the content has an invalid structure
|
|
ErrInvalidStructure = errors.New("Content has an invalid structure")
|
|
|
|
// ErrInvalidOIDFormat occurs if the oid has an invalid format
|
|
ErrInvalidOIDFormat = errors.New("OID has an invalid format")
|
|
)
|
|
|
|
// ReadPointer tries to read LFS pointer data from the reader
|
|
func ReadPointer(reader io.Reader) (Pointer, error) {
|
|
buf := make([]byte, blobSizeCutoff)
|
|
n, err := io.ReadFull(reader, buf)
|
|
if err != nil && err != io.ErrUnexpectedEOF {
|
|
return Pointer{}, err
|
|
}
|
|
buf = buf[:n]
|
|
|
|
return ReadPointerFromBuffer(buf)
|
|
}
|
|
|
|
var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
|
|
|
|
// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
|
|
func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
|
|
var p Pointer
|
|
|
|
headString := string(buf)
|
|
if !strings.HasPrefix(headString, MetaFileIdentifier) {
|
|
return p, ErrMissingPrefix
|
|
}
|
|
|
|
splitLines := strings.Split(headString, "\n")
|
|
if len(splitLines) < 3 {
|
|
return p, ErrInvalidStructure
|
|
}
|
|
|
|
oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
|
|
if len(oid) != 64 || !oidPattern.MatchString(oid) {
|
|
return p, ErrInvalidOIDFormat
|
|
}
|
|
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
|
|
p.Oid = oid
|
|
p.Size = size
|
|
|
|
return p, nil
|
|
}
|
|
|
|
// IsValid checks if the pointer has a valid structure.
|
|
// It doesn't check if the pointed-to-content exists.
|
|
func (p Pointer) IsValid() bool {
|
|
if len(p.Oid) != 64 {
|
|
return false
|
|
}
|
|
if !oidPattern.MatchString(p.Oid) {
|
|
return false
|
|
}
|
|
if p.Size < 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// StringContent returns the string representation of the pointer
|
|
// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
|
|
func (p Pointer) StringContent() string {
|
|
return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size)
|
|
}
|
|
|
|
// RelativePath returns the relative storage path of the pointer
|
|
func (p Pointer) RelativePath() string {
|
|
if len(p.Oid) < 5 {
|
|
return p.Oid
|
|
}
|
|
|
|
return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
|
|
}
|
|
|
|
// GeneratePointer generates a pointer for arbitrary content
|
|
func GeneratePointer(content io.Reader) (Pointer, error) {
|
|
h := sha256.New()
|
|
c, err := io.Copy(h, content)
|
|
if err != nil {
|
|
return Pointer{}, err
|
|
}
|
|
sum := h.Sum(nil)
|
|
return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil
|
|
}
|