Add support for thumbnailing animated webps, both to animated thumbna…

Dockerfile

@@ -19,7 +19,9 @@ COPY --from=builder /opt/bin/media_repo /opt/bin/import_synapse /opt/bin/gdpr_ex
 RUN apk add --no-cache \
         su-exec \
         ca-certificates \
-        dos2unix
+        dos2unix \
+        libwebp \
+        libwebp-tools
 
 COPY ./config.sample.yaml /etc/media-repo.yaml.sample
 COPY ./docker/run.sh /usr/local/bin/

config.sample.yaml

@@ -391,6 +391,8 @@ thumbnails:
     - "image/png"
     - "image/gif"
     - "image/heif"
+    # Thumbnailing animated webps as still frames requires either libwebp or ImageMagick compiled with --with-webp
+    # Thumbnailing animated webps as animated webps requires both ffmpeg and ImageMagick compiled with --with-webp
     - "image/webp"
     #- "image/svg+xml" # Be sure to have ImageMagick installed to thumbnail SVG files
     - "audio/mpeg"
@@ -546,4 +548,4 @@ featureSupport:
       - name: "server2"
         addr: ":7001"
       - name: "server3"
-        addr: ":7002"
+        addr: ":7002"

thumbnailing/i/animated_webp.go

@@ -0,0 +1,139 @@
+package i
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+
+	"github.com/disintegration/imaging"
+	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
+	"github.com/turt2live/matrix-media-repo/thumbnailing/u"
+	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/cleanup"
+)
+
+type animatedWebpGenerator struct {
+}
+
+func (d animatedWebpGenerator) supportedContentTypes() []string {
+	return []string{"image/webp"}
+}
+
+func (d animatedWebpGenerator) supportsAnimation() bool {
+	return true
+}
+
+func (d animatedWebpGenerator) matches(img []byte, contentType string) bool {
+	return contentType == "image/webp" && util.IsAnimatedWebp(img)
+}
+
+func (d animatedWebpGenerator) GenerateThumbnail(b []byte, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) {
+	if !animated {
+		return webpGenerator{}.GenerateThumbnail(b, "image/webp", width, height, method, false, ctx)
+	}
+
+	key, err := util.GenerateRandomString(16)
+	if err != nil {
+		return nil, errors.New("animated webp: error generating temp key: " + err.Error())
+	}
+	sourceFile := path.Join(os.TempDir(), "media_repo."+key+".1.webp")
+	videoFile := path.Join(os.TempDir(), "media_repo."+key+".2.mp4")
+	outFile := path.Join(os.TempDir(), "media_repo."+key+".3.webp")
+	frameFile := path.Join(os.TempDir(), "media_repo."+key+".3.png")
+	defer os.Remove(sourceFile)
+	defer os.Remove(videoFile)
+	defer os.Remove(outFile)
+	defer os.Remove(frameFile)
+
+	f, err := os.OpenFile(sourceFile, os.O_RDWR|os.O_CREATE, 0640)
+	if err != nil {
+		return nil, errors.New("animated webp: error writing temp webp file: " + err.Error())
+	}
+	_, _ = f.Write(b)
+	cleanup.DumpAndCloseStream(f)
+
+	// we need to fetch the first frame to be able to get source width / height
+	err = exec.Command("convert", sourceFile+"[0]", frameFile).Run()
+	if err != nil {
+		return nil, errors.New("animated webp: error decoding webp first frame: " + err.Error())
+	}
+	b, err = ioutil.ReadFile(frameFile)
+	if err != nil {
+		return nil, errors.New("animated webp: error reading first frame: " + err.Error())
+	}
+	src, err := imaging.Decode(bytes.NewBuffer(b))
+	if err != nil {
+		return nil, errors.New("animated webp: error decoding png first frame: " + err.Error())
+	}
+
+	var shouldThumbnail bool
+	shouldThumbnail, width, height, _, method = u.AdjustProperties(src, width, height, false, false, method)
+	if !shouldThumbnail {
+		return nil, nil
+	}
+
+	err = exec.Command("convert", sourceFile, videoFile).Run()
+	if err != nil {
+		return nil, errors.New("animated webp: error converting webp file to mp4 file: " + err.Error())
+	}
+
+	srcWidth := src.Bounds().Max.X
+	srcHeight := src.Bounds().Max.Y
+	aspectRatio := float32(srcWidth) / float32(srcHeight)
+	targetAspectRatio := float32(width) / float32(height)
+	if method == "scale" {
+		// ffmpeg -i out.mp4 -vf scale=width:heigth out.webp
+		scaleWidth := width
+		scaleHeight := height
+		if targetAspectRatio < aspectRatio {
+			// height needs increasing
+			scaleHeight = int(float32(scaleWidth) * aspectRatio)
+		} else {
+			// width needs increasing
+			scaleWidth = int(float32(scaleHeight) * aspectRatio)
+		}
+		err = exec.Command("ffmpeg", "-i", videoFile, "-vf", "scale=" + strconv.Itoa(scaleWidth) + ":" + strconv.Itoa(scaleHeight), outFile).Run()
+	} else if method == "crop" {
+		// ffmpeg -i out.mp4 -vf crop=iw-400:ih-40,scale=960:720 out.webp
+		cropWidth := "iw"
+		cropHeight := "ih"
+		if targetAspectRatio < aspectRatio {
+			// width needs cropping
+			newWidth := float32(srcWidth) * targetAspectRatio / aspectRatio
+			cropWidth = strconv.Itoa(int(newWidth))
+		} else {
+			// height needs cropping
+			newHeight := float32(srcWidth) * aspectRatio / targetAspectRatio
+			cropHeight = strconv.Itoa(int(newHeight))
+		}
+		err = exec.Command("ffmpeg", "-i", videoFile, "-vf", "crop=" + cropWidth + ":" + cropHeight + ",scale=" + strconv.Itoa(width) + ":" + strconv.Itoa(height), outFile).Run()
+	} else {
+		return nil, errors.New("animated webp: unrecognized method: " + method)
+	}
+	if err != nil {
+		return nil, errors.New("animated webp: error scaling/cropping file: " + err.Error())
+	}
+	// set the animation to infinite loop again
+	err = exec.Command("convert", outFile, "-loop", "0", outFile).Run()
+	if err != nil {
+		return nil, errors.New("animated webp: error setting webp to loop: " + err.Error())
+	}
+	b, err = ioutil.ReadFile(outFile)
+	if err != nil {
+		return nil, errors.New("animated webp: error reading resulting webp thumbnail: " + err.Error())
+	}
+	return &m.Thumbnail{
+		Animated:    true,
+		ContentType: "image/webp",
+		Reader:      ioutil.NopCloser(bytes.NewReader(b)),
+	}, nil
+}
+
+func init() {
+	generators = append(generators, animatedWebpGenerator{})
+}

thumbnailing/i/webp.go

@@ -3,9 +3,15 @@ package i
 import (
 	"bytes"
 	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
 
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
+	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/cleanup"
 	"golang.org/x/image/webp"
 )
 
@@ -17,7 +23,7 @@ func (d webpGenerator) supportedContentTypes() []string {
 }
 
 func (d webpGenerator) supportsAnimation() bool {
-	return true
+	return false
 }
 
 func (d webpGenerator) matches(img []byte, contentType string) bool {
@@ -27,7 +33,50 @@ func (d webpGenerator) matches(img []byte, contentType string) bool {
 func (d webpGenerator) GenerateThumbnail(b []byte, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) {
 	src, err := webp.Decode(bytes.NewBuffer(b))
 	if err != nil {
-		return nil, errors.New("webp: error decoding thumbnail: " + err.Error())
+		// the decoder isn't able to read all webp files. So, if it failed, we'll re-try with libwebp
+		nativeDecodeError := err.Error()
+
+		key, err := util.GenerateRandomString(16)
+		if err != nil {
+			return nil, errors.New("webp: error decoding thumbnail: " + nativeDecodeError + ", error generating temp key: " + err.Error())
+		}
+		tempFile1 := path.Join(os.TempDir(), "media_repo."+key+".1.webp")
+		tempFile2 := path.Join(os.TempDir(), "media_repo."+key+".2.webp")
+		tempFile3 := path.Join(os.TempDir(), "media_repo."+key+".3.png")
+		defer os.Remove(tempFile1)
+		defer os.Remove(tempFile2)
+		defer os.Remove(tempFile3)
+
+		f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640)
+		if err != nil {
+			return nil, errors.New("webp: error decoding thumbnail: " + nativeDecodeError + ", error writing temp webp file: " + err.Error())
+		}
+		_, _ = f.Write(b)
+		cleanup.DumpAndCloseStream(f)
+
+		err = exec.Command("dwebp", tempFile1, "-o", tempFile3).Run()
+		if err != nil {
+			// the command failed, meaning the webp might have been animated. So, we
+			// extrac tthe frame first and then try again
+			err = exec.Command("webpmux", "-get", "frame", "1", tempFile1, "-o", tempFile2).Run()
+			if err == nil {
+				err = exec.Command("dwebp", tempFile2, "-o", tempFile3).Run()
+			}
+			if err != nil {
+				// try via convert binary
+				err = exec.Command("convert", tempFile1+"[0]", tempFile3).Run()
+			}
+			if err != nil {
+				return nil, errors.New("webp: error decoding thumbnail: " + nativeDecodeError + ", error converting webp file: " + err.Error())
+			}
+		}
+
+		b, err = ioutil.ReadFile(tempFile3)
+		if err != nil {
+			return nil, errors.New("webp: error decoding thumbnail: " + nativeDecodeError + ", error reading temp png file: " + err.Error())
+		}
+
+		return pngGenerator{}.GenerateThumbnail(b, "image/png", width, height, method, false, ctx)
 	}
 
 	return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx)

util/imaging.go

@@ -28,3 +28,19 @@ func IsAnimatedPNG(b []byte) bool {
 
 	return false
 }
+
+func IsAnimatedWebp(b []byte) bool {
+	// https://stackoverflow.com/a/61242086
+	// first we validate the header
+	header := []byte("VP8X")
+	i := 0
+	for i < 4 {
+		if b[12 + i] != header[i] {
+			return false
+		}
+		i++
+	}
+	// now we validate the flag
+	flagByte := b[20];
+	return ((flagByte >> 1) & 1) > 0
+}