Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow configuring per-user limits on upload size. #385

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions api/r0/public_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,15 @@ import (

"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/quota"
)

type PublicConfigResponse struct {
UploadMaxSize int64 `json:"m.upload.size,omitempty"`
}

func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
uploadSize := rctx.Config.Uploads.ReportedMaxSizeBytes
if uploadSize == 0 {
uploadSize = rctx.Config.Uploads.MaxSizeBytes
}

if uploadSize < 0 {
uploadSize = 0 // invokes the omitEmpty
}

uploadSize := quota.GetUserUploadMaxSizeBytes(rctx, user.UserId)
return &PublicConfigResponse{
UploadMaxSize: uploadSize,
}
Expand Down
5 changes: 3 additions & 2 deletions api/r0/upload.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package r0

import (
"github.com/getsentry/sentry-go"
"io"
"io/ioutil"
"net/http"
"path/filepath"

"github.com/getsentry/sentry-go"

"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/common"
Expand Down Expand Up @@ -35,7 +36,7 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf
contentType = "application/octet-stream" // binary
}

if upload_controller.IsRequestTooLarge(r.ContentLength, r.Header.Get("Content-Length"), rctx) {
if upload_controller.IsRequestTooLarge(r.ContentLength, r.Header.Get("Content-Length"), rctx, user.UserId) {
io.Copy(ioutil.Discard, r.Body) // Ditch the entire request
return api.RequestTooLarge()
}
Expand Down
9 changes: 5 additions & 4 deletions common/config/models_domain.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ type QuotasConfig struct {
}

type UploadsConfig struct {
MaxSizeBytes int64 `yaml:"maxBytes"`
MinSizeBytes int64 `yaml:"minBytes"`
ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"`
Quota QuotasConfig `yaml:"quotas"`
MaxSizeBytes int64 `yaml:"maxBytes"`
UsersMaxSizeBytes []QuotaUserConfig `yaml:"quotas,flow"`
MinSizeBytes int64 `yaml:"minBytes"`
ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"`
Quota QuotasConfig `yaml:"quotas"`
}

type DatastoreConfig struct {
Expand Down
4 changes: 4 additions & 0 deletions config.sample.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ uploads:
# The maximum individual file size a user can upload.
maxBytes: 104857600 # 100MB default, 0 to disable

usersMaxBytes:
- glob: "@*:*" # Affect all users. Use asterisks (*) to match any character.
maxBytes: 104857600 # 100MB default, 0 to disable

# The minimum number of bytes to let people upload. This is recommended to be non-zero to
# ensure that the "cost" of running the media repo is worthwhile - small file uploads tend
# to waste more CPU and database resources than small files, thus a default of 100 bytes
Expand Down
5 changes: 3 additions & 2 deletions controllers/preview_controller/preview_resource_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ package preview_controller

import (
"fmt"
"github.com/getsentry/sentry-go"
"sync"

"github.com/getsentry/sentry-go"

"github.com/disintegration/imaging"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
Expand Down Expand Up @@ -129,7 +130,7 @@ func urlPreviewWorkFn(request *resource_handler.WorkRequest) (resp *urlPreviewRe
}

// Store the thumbnail, if there is one
if preview.Image != nil && !upload_controller.IsRequestTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader, ctx) {
if preview.Image != nil && !upload_controller.IsRequestTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader, ctx, info.forUserId) {
contentLength := upload_controller.EstimateContentLength(preview.Image.ContentLength, preview.Image.ContentLengthHeader)

// UploadMedia will close the read stream for the thumbnail and dedupe the image
Expand Down
13 changes: 8 additions & 5 deletions controllers/upload_controller/upload_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@ package upload_controller

import (
"fmt"
"github.com/getsentry/sentry-go"
"io"
"io/ioutil"
"strconv"
"time"

"github.com/getsentry/sentry-go"

"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/plugins"
"github.com/turt2live/matrix-media-repo/quota"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/datastore"
"github.com/turt2live/matrix-media-repo/types"
Expand All @@ -32,12 +34,13 @@ type AlreadyUploadedFile struct {
ObjectInfo *types.ObjectInfo
}

func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcontext.RequestContext) bool {
if ctx.Config.Uploads.MaxSizeBytes <= 0 {
func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcontext.RequestContext, userId string) bool {
maxSize := quota.GetUserUploadMaxSizeBytes(ctx, userId)
if maxSize <= 0 {
return false
}
if contentLength >= 0 {
return contentLength > ctx.Config.Uploads.MaxSizeBytes
return contentLength > maxSize
}
if contentLengthHeader != "" {
parsed, err := strconv.ParseInt(contentLengthHeader, 10, 64)
Expand All @@ -47,7 +50,7 @@ func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcon
return true // Invalid header
}

return parsed > ctx.Config.Uploads.MaxSizeBytes
return parsed > maxSize
}

return false // We can only assume
Expand Down
16 changes: 16 additions & 0 deletions quota/quota.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,19 @@ func IsUserWithinQuota(ctx rcontext.RequestContext, userId string) (bool, error)

return true, nil // no rules == no quota
}

func GetUserUploadMaxSizeBytes(ctx rcontext.RequestContext, userId string) int64 {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could probably easily go in upload/upload.go, but got lazy. Shout if this function needs moving.

var maxValue int64 = 0
for _, q := range ctx.Config.Uploads.UsersMaxSizeBytes {
if glob.Glob(q.Glob, userId) {
if q.MaxBytes > maxValue {
maxValue = q.MaxBytes
}
}
}
if maxValue == 0 {
return ctx.Config.Uploads.MaxSizeBytes
} else {
return maxValue
}
}