Clipshow: Media (images and videos) to side-by-side movie

Main.kt

/**
 * @author Benjamin Hill benjaminhill@gmail.com
 */

import mu.KotlinLogging
import net.coobird.thumbnailator.Thumbnails
import org.bytedeco.javacpp.avutil
import org.bytedeco.javacpp.avutil.av_log_set_level
import org.bytedeco.javacv.FFmpegFrameGrabber
import org.bytedeco.javacv.FFmpegFrameRecorder
import org.bytedeco.javacv.FrameConverter
import org.bytedeco.javacv.Java2DFrameConverter
import java.awt.image.BufferedImage
import java.io.File
import javax.imageio.ImageIO

val LOG = KotlinLogging.logger {}

const val RES_WIDTH = 1280
const val RES_HEIGHT = 720

val ROOT = System.getProperty("user.home") + "/Desktop/arts_focus"

/**
 * Make a slideshow
 * Read through a folder of ./clips (movies or images)
 * Pack them into the left or right side of a 720p output video
 * Customizations: credits.png at the beginning, ./fullscreen at the end
 * Great to use the mp4 videos that come from Live Motion
 */
fun main() {
    av_log_set_level(org.bytedeco.javacpp.avutil.AV_LOG_ERROR) // https://github.com/bytedeco/javacv/issues/780

    FFmpegFrameRecorder("$ROOT/out.mp4", RES_WIDTH, RES_HEIGHT, 0).apply {
        frameRate = 30.0
        videoBitrate = 0 //10_000_000 // 0=max
        videoQuality = 0.0 // 10.0 // 0.0 = max?  @see https://trac.ffmpeg.org/wiki/Encode/H.264
        start()
    }.use { ffr ->
        LOG.info { "Starting recording to 'out.mp4' (${ffr.imageWidth}, ${ffr.imageHeight})" }
        var frameCount = 0

        fileToImages(File("$ROOT/credits.png")).let { nextClip ->
            LOG.info { "$frameCount clip into leftPortrait (as fullscreen credits)" }
            while (nextClip.hasNext()) {
                drawFrame(nextClip, emptyList<BufferedImage>().iterator(), ffr)
                frameCount++
            }
        }

        var leftSlot = emptyList<BufferedImage>().iterator()
        var rightSlot = emptyList<BufferedImage>().iterator()

        File("$ROOT/clips/")
                .walk()
                .filter { it.isFile && it.canRead() && !it.isHidden }
                .sortedBy { it.name }
                .map { fileToImages(it) }
                .forEach { nextClip ->
                    // While all slots are full AND have content, loop and render
                    while (leftSlot.hasNext() && rightSlot.hasNext()) {
                        drawFrame(leftSlot, rightSlot, ffr)
                        frameCount++
                    }

                    // Drop in first free slot
                    if (!leftSlot.hasNext()) {
                        leftSlot = nextClip
                        LOG.info { "$frameCount clip into leftPortrait" }
                    } else if (!rightSlot.hasNext()) {
                        rightSlot = nextClip
                        LOG.info { "$frameCount clip into rightPortrait" }
                    } else {
                        LOG.warn { "Why are both areas full?" }
                    }
                }

        // Finish out remaining sequences
        while (leftSlot.hasNext() || rightSlot.hasNext()) {
            drawFrame(leftSlot, rightSlot, ffr)
        }

        LOG.info { "Done with half, now drawing full frames" }

        File("$ROOT/fullscreen/")
                .walk()
                .filter { it.isFile && it.canRead() && !it.isHidden }
                .sortedBy { it.name }
                .map { fileToImages(it) }
                .forEach { nextClip ->
                    LOG.info { "$frameCount clip into leftPortrait (as fullscreen)" }
                    while (nextClip.hasNext()) {
                        drawFrame(nextClip, emptyList<BufferedImage>().iterator(), ffr)
                        frameCount++
                    }
                }

        LOG.info { "Total $frameCount frames" }
        ffr.stop()
    }
}

/**
 * Also `mogrify -auto-orient -path ../rotated *.jpg`
 */
fun BufferedImage.rotateClockwise90() = BufferedImage(height, width, type).also { dest ->
    dest.createGraphics().let { g2d ->
        g2d.translate((height - width) / 2, (height - width) / 2)
        g2d.rotate(Math.PI / 2, (height / 2).toDouble(), (width / 2).toDouble())
        g2d.drawRenderedImage(this, null)
        g2d.dispose()
    }
}

private val converter = object : ThreadLocal<FrameConverter<BufferedImage>>() {
    override fun initialValue() = Java2DFrameConverter()
}

/** If fullscreen candidate, return big else return half size */
fun BufferedImage.smallify(): BufferedImage = if ((width == 1920 && height == 1080) ||
        (width == 1280 && height == 720)) {
    Thumbnails.of(this).size(RES_WIDTH, RES_HEIGHT).asBufferedImage()!!
} else {
    Thumbnails.of(this).size(RES_WIDTH / 2, RES_HEIGHT).asBufferedImage()!!
}

/** If a movie then direct to small images, if an image then copy to 2 seconds of images */
fun fileToImages(file: File): Iterator<BufferedImage> = iterator {
    when (file.extension.toLowerCase()) {
        "jpg", "jpeg", "png" -> {
            // Doesn't respect rotation: val bi = ImageIO.read(file)!!
            val bi = ImageIO.read(file).smallify()
            // 2 seconds same as clips
            repeat(30 * 2) { yield(bi) }
        }
        "mp4", "mpeg", "mov" -> FFmpegFrameGrabber(file).use { grabber ->
            grabber.start()
            val rotation = (grabber.getVideoMetadata("rotate") ?: "0").toInt()

            LOG.debug { "Started getting frames from ${file.name} rotation:$rotation" }
            while (true) {
                yield(grabber.grabImage()?.let { nextFrame ->
                    val rotated = converter.get().convert(nextFrame)
                            //.deepCopy()
                            .let { bi ->
                                when (rotation) {
                                    0 -> bi
                                    90 -> bi.rotateClockwise90()
                                    else -> bi
                                }
                            }
                    rotated.smallify()
                } ?: break)
            }
            grabber.stop()
        }
        else -> {
            LOG.warn { "Don't know how to handle ${file.name}" }
        }
    }
}

fun drawFrame(
        leftPortrait: Iterator<BufferedImage>,
        rightPortrait: Iterator<BufferedImage>,
        ffr: FFmpegFrameRecorder) {
    val frame = BufferedImage(ffr.imageWidth, ffr.imageHeight, BufferedImage.TYPE_INT_ARGB)
    frame.createGraphics()!!.apply {
        if (leftPortrait.hasNext()) {
            val bi = leftPortrait.next()
            // Center in your side OR be fullscreen
            val dx = if (bi.width < RES_WIDTH) {
                (RES_WIDTH / 2 - bi.width) / 2
            } else {
                0
            }
            val dy = (RES_HEIGHT - bi.height) / 2
            drawImage(bi, dx, dy, null)
        }
        if (rightPortrait.hasNext()) {
            // Center in your side
            val bi = rightPortrait.next()
            val dx = (RES_WIDTH / 2 - bi.width) / 2
            val dy = (RES_HEIGHT - bi.height) / 2
            drawImage(bi, RES_WIDTH / 2 + dx, dy, null)
        }
        dispose()
    }
    ffr.record(converter.get().convert(frame), avutil.AV_PIX_FMT_ARGB)
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xmlns="http://maven.apache.org/POM/4.0.0"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>info.benjaminhill</groupId>
    <artifactId>makeslideshow</artifactId>
    <packaging>pom</packaging>
    <version>0.0.1-SNAPSHOT</version>
    <name>Make a Slideshow</name>
    <url>https://github.com/salamanders/makeslideshow</url>

    <properties>
        <kotlin.version>1.3.21</kotlin.version>
        <kotlin.coroutine.version>1.1.1</kotlin.coroutine.version>
        <kotlin.compiler.languageVersion>1.3</kotlin.compiler.languageVersion>
        <kotlin.compiler.jvmTarget>1.8</kotlin.compiler.jvmTarget>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.jetbrains.kotlin</groupId>
            <artifactId>kotlin-stdlib-jdk8</artifactId>
            <version>${kotlin.version}</version>
        </dependency>

        <!-- Logging -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.8.0-beta4</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.8.0-beta4</version>
        </dependency>
        <dependency>
            <groupId>io.github.microutils</groupId>
            <artifactId>kotlin-logging</artifactId>
            <version>1.6.25</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.5</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-coroutines-core -->
        <dependency>
            <groupId>org.jetbrains.kotlinx</groupId>
            <artifactId>kotlinx-coroutines-core</artifactId>
            <version>1.2.0-alpha</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.bytedeco/javacv -->
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>javacv</artifactId>
            <version>1.4.4</version>
        </dependency>
        <dependency>
            <groupId>org.bytedeco.javacpp-presets</groupId>
            <artifactId>opencv</artifactId>
            <version>4.0.1-1.4.4</version>
        </dependency>

        <dependency>
            <groupId>org.bytedeco.javacpp-presets</groupId>
            <artifactId>ffmpeg-platform</artifactId>
            <version>4.1-1.4.4</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/net.coobird/thumbnailator -->
        <dependency>
            <groupId>net.coobird</groupId>
            <artifactId>thumbnailator</artifactId>
            <version>0.4.8</version>
        </dependency>


        <!-- Meta: checking for updates `mvn versions:display-dependency-updates` -->
        <dependency>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>versions-maven-plugin</artifactId>
            <version>2.7</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.jetbrains.kotlin</groupId>
                <artifactId>kotlin-maven-plugin</artifactId>
                <version>${kotlin.version}</version>
                <executions>
                    <execution>
                        <id>compile</id>
                        <phase>compile</phase>
                        <goals>
                            <goal>compile</goal>
                        </goals>
                        <configuration>
                            <sourceDirs>
                                <sourceDir>${project.basedir}/src/main/kotlin</sourceDir>
                            </sourceDirs>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>


</project>