studio/modules/storage/storage.go

// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package storage

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"path"

	"code.gitea.io/gitea/modules/httplib"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/public"
	"code.gitea.io/gitea/modules/setting"
)

// ErrURLNotSupported represents url is not supported
var ErrURLNotSupported = errors.New("url method not supported")

// ErrInvalidConfiguration is called when there is invalid configuration for a storage
type ErrInvalidConfiguration struct {
	cfg any
	err error
}

func (err ErrInvalidConfiguration) Error() string {
	if err.err != nil {
		return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err)
	}
	return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg)
}

// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
func IsErrInvalidConfiguration(err error) bool {
	_, ok := err.(ErrInvalidConfiguration)
	return ok
}

type Type = setting.StorageType

// NewStorageFunc is a function that creates a storage
type NewStorageFunc func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)

var storageMap = map[Type]NewStorageFunc{}

// RegisterStorageType registers a provided storage type with a function to create it
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)) {
	storageMap[typ] = fn
}

// Object represents the object on the storage
type Object interface {
	io.ReadCloser
	io.Seeker
	Stat() (os.FileInfo, error)
}

// ServeDirectOptions customizes HTTP headers for a generated signed URL.
type ServeDirectOptions struct {
	// Overrides the automatically detected MIME type.
	ContentType string
}

// Safe defaults are applied only when not explicitly overridden by the caller.
func prepareServeDirectOptions(optsOptional *ServeDirectOptions, name string) (ret struct {
	ContentType        string
	ContentDisposition string
},
) {
	// Here we might not know the real filename, and it's quite inefficient to detect the MIME type by pre-fetching the object head.
	// So we just do a quick detection by extension name, at least it works for the "View Raw File" for an LFS file on the Web UI.
	// TODO: OBJECT-STORAGE-CONTENT-TYPE: need a complete solution and refactor for Azure in the future

	if optsOptional != nil {
		ret.ContentType = optsOptional.ContentType
	}
	name = path.Base(name)
	if ret.ContentType == "" {
		ext := path.Ext(name)
		ret.ContentType = public.DetectWellKnownMimeType(ext)
	}
	// When using ServeDirect, the URL is from the object storage's web server,
	// it is not the same origin as Gitea server, so it should be safe enough to use "inline" to render the content directly.
	// If a browser doesn't support the content type to be displayed inline, browser will download with the filename.
	ret.ContentDisposition = httplib.EncodeContentDispositionInline(name)
	return ret
}

// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
	Open(path string) (Object, error)

	// Save store an object, if size is unknown set -1
	// NOTICE: Some storage SDK will close the Reader after saving if it is also a Closer,
	// DO NOT use the reader anymore after Save, or wrap it to a non-Closer reader.
	Save(path string, r io.Reader, size int64) (int64, error)

	Stat(path string) (os.FileInfo, error)
	Delete(path string) error

	// ServeDirectURL generates a "serve-direct" URL for the specified blob storage file,
	// end user (browser) will use this URL to access the file directly from the object storage, bypassing Gitea server.
	// Usually the link is time-limited (a few minutes) and contains a signature to ensure security.
	// The generated URL must NOT use the same origin as Gitea server, otherwise it will cause security issues.
	// * method defines which HTTP method is permitted for certain storage providers (e.g., MinIO).
	// * opt allows customizing the Content-Type and Content-Disposition headers.
	// TODO: need to merge "ServeDirect()" check into this function, avoid duplicate code and potential inconsistency.
	ServeDirectURL(path, name, method string, opt *ServeDirectOptions) (*url.URL, error)

	// IterateObjects calls the iterator function for each object in the storage with the given path as prefix
	// The "fullPath" argument in callback is the full path in this storage.
	// * IterateObjects("", ...): iterate all objects in this storage
	// * IterateObjects("sub-path", ...): iterate all objects with "sub-path" as prefix in this storage, the "fullPath" will be like "sub-path/xxx"
	IterateObjects(basePath string, iterator func(fullPath string, obj Object) error) error
}

// Copy copies a file from source ObjectStorage to dest ObjectStorage
func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
	f, err := srcStorage.Open(srcPath)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	size := int64(-1)
	fsinfo, err := f.Stat()
	if err == nil {
		size = fsinfo.Size()
	}

	return dstStorage.Save(dstPath, f, size)
}

// Clean delete all the objects in this storage
func Clean(storage ObjectStorage) error {
	return storage.IterateObjects("", func(path string, obj Object) error {
		_ = obj.Close()
		return storage.Delete(path)
	})
}

// SaveFrom saves data to the ObjectStorage with path p from the callback
func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) error) error {
	pr, pw := io.Pipe()
	defer pr.Close()
	go func() {
		defer pw.Close()
		if err := callback(pw); err != nil {
			_ = pw.CloseWithError(err)
		}
	}()

	_, err := objStorage.Save(path, pr, -1)
	return err
}

var (
	// Attachments represents attachments storage
	Attachments ObjectStorage = uninitializedStorage

	// LFS represents lfs storage
	LFS ObjectStorage = uninitializedStorage

	// Avatars represents user avatars storage
	Avatars ObjectStorage = uninitializedStorage
	// RepoAvatars represents repository avatars storage
	RepoAvatars ObjectStorage = uninitializedStorage

	// RepoArchives represents repository archives storage
	RepoArchives ObjectStorage = uninitializedStorage

	// Packages represents packages storage
	Packages ObjectStorage = uninitializedStorage

	// Actions represents actions storage
	Actions ObjectStorage = uninitializedStorage
	// ActionsArtifacts Artifacts represents actions artifacts storage
	ActionsArtifacts ObjectStorage = uninitializedStorage

	// HTMLPreview represents HTML preview storage (S3 bucket for serving HTML files)
	HTMLPreview ObjectStorage = uninitializedStorage
)

// Init init the storage
func Init() error {
	for _, f := range []func() error{
		initAttachments,
		initAvatars,
		initRepoAvatars,
		initLFS,
		initRepoArchives,
		initPackages,
		initActions,
		initHTMLPreview,
	} {
		if err := f(); err != nil {
			return err
		}
	}
	return nil
}

// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
func NewStorage(typStr Type, cfg *setting.Storage) (ObjectStorage, error) {
	if len(typStr) == 0 {
		typStr = setting.LocalStorageType
	}
	fn, ok := storageMap[typStr]
	if !ok {
		return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
	}

	return fn(context.Background(), cfg)
}

func initAvatars() (err error) {
	log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type)
	Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage)
	return err
}

func initAttachments() (err error) {
	if !setting.Attachment.Enabled {
		Attachments = discardStorage("Attachment isn't enabled")
		return nil
	}
	log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type)
	Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage)
	return err
}

func initLFS() (err error) {
	if !setting.LFS.StartServer {
		LFS = discardStorage("LFS isn't enabled")
		return nil
	}
	log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type)
	LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage)
	return err
}

func initRepoAvatars() (err error) {
	log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type)
	RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage)
	return err
}

func initRepoArchives() (err error) {
	log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
	RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, setting.RepoArchive.Storage)
	return err
}

func initPackages() (err error) {
	if !setting.Packages.Enabled {
		Packages = discardStorage("Packages isn't enabled")
		return nil
	}
	log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type)
	Packages, err = NewStorage(setting.Packages.Storage.Type, setting.Packages.Storage)
	return err
}

func initActions() (err error) {
	if !setting.Actions.Enabled {
		Actions = discardStorage("Actions isn't enabled")
		ActionsArtifacts = discardStorage("ActionsArtifacts isn't enabled")
		return nil
	}
	log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
	if Actions, err = NewStorage(setting.Actions.LogStorage.Type, setting.Actions.LogStorage); err != nil {
		return err
	}
	log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
	ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, setting.Actions.ArtifactStorage)
	return err
}

func initHTMLPreview() (err error) {
	if !setting.HTMLPreview.Enabled {
		HTMLPreview = discardStorage("HTMLPreview isn't enabled")
		return nil
	}
	log.Info("Initialising HTMLPreview storage with type: %s", setting.HTMLPreview.Storage.Type)
	HTMLPreview, err = NewStorage(setting.HTMLPreview.Storage.Type, setting.HTMLPreview.Storage)
	return err
}