A java sketch

GPhotoUploader.java

package net.digihippo.gphoto;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class GPhotoUploader {
    interface Client
    {
        // We'd expect to be able to call this from multiple threads
        void upload(
                Path file,
                String albumTitle,
                Consumer<UploadedPhoto> onSuccess
        );

        // These two probably aren't thread safe
        String ensureAlbumExists(
                String albumTitle,
                Consumer<AlbumCreated> onAlbumCreated);

        void assignMediaItems(
                Optional<String> albumId,
                List<UploadedPhoto> value,
                Consumer<List<Path>> itemsAssigned
        );
    }

    // Implementations probably ought to be thread safe
    interface Database
    {
        boolean requiresUpload(Path path);

        // This, at least, is likely to be called from multiple worker threads
        void itemUploaded(UploadedPhoto uploadedPhoto);

        // Everything that's been uploaded but has not been `batchItemCreated`
        Stream<UploadedPhoto> requiringMediaItemCreation();

        // Mark these items as having been `batchItemCreated`
        void batchItemsCreated(List<Path> paths);

        // Not essential yet
        void albumCreated(AlbumCreated albumCreated);
    }

    public static final class UploadedPhoto
    {
        public final Path file;
        public final String albumTitle;
        public final String uploadToken;

        public UploadedPhoto(
                Path file,
                String albumTitle,
                String uploadToken) {
            this.file = file;
            this.albumTitle = albumTitle;
            this.uploadToken = uploadToken;
        }
    }

    public static final class AlbumCreated
    {
        public final String albumTitle;
        public final String albumId;

        public AlbumCreated(String albumTitle, String albumId) {
            this.albumTitle = albumTitle;
            this.albumId = albumId;
        }
    }

    private static final class InMemoryDatabase implements Database
    {
        private static final class StoredMediaItem
        {
            public final UploadedPhoto photo;
            public boolean assigned = false;

            public StoredMediaItem(UploadedPhoto photo) {
                this.photo = photo;
            }

            void onAssignment()
            {
                this.assigned = true;
            }
        }

        // Unused right now. We could initialise this at startup, and then
        // not need Client::ensureAlbumExists
        @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
        private final Map<String, String> albumTitleToAlbumId =
                new HashMap<>();

        private final Map<Path, StoredMediaItem> store = new HashMap<>();

        @Override
        public boolean requiresUpload(Path path) {
            return !store.containsKey(path);
        }

        @Override
        public void itemUploaded(UploadedPhoto uploadedPhoto) {
            store.put(uploadedPhoto.file, new StoredMediaItem(uploadedPhoto));
        }

        @Override
        public void batchItemsCreated(List<Path> mediaItems) {
            mediaItems.forEach(mi -> store.get(mi).onAssignment());
        }

        @Override
        public void albumCreated(AlbumCreated albumCreated) {
            albumTitleToAlbumId.put(
                    albumCreated.albumTitle,
                    albumCreated.albumId);
        }

        @Override
        public Stream<UploadedPhoto> requiringMediaItemCreation() {
            return store
                    .values()
                    .stream()
                    .filter(smi -> !smi.assigned)
                    .map(smi -> smi.photo);
        }
    }

    static class DryRunClient implements Client
    {
        private final Set<String> createdAlbums = new HashSet<>();

        @Override
        public void upload(
                Path file,
                String albumTitle,
                Consumer<UploadedPhoto> onSuccess) {
            System.out.println("Uploading " + file);
            onSuccess.accept(new UploadedPhoto(file, albumTitle, file.toString()));
        }

        @Override
        public String ensureAlbumExists(
                String albumTitle,
                Consumer<AlbumCreated> onAlbumCreated
        ) {
            if (createdAlbums.add(albumTitle))
            {
                System.out.println("Creating " + albumTitle);
                onAlbumCreated.accept(new AlbumCreated(
                        albumTitle,
                        albumTitle
                ));
            }
            else
            {
                System.out.println("Album " + albumTitle + " already exists");
            }

            return albumTitle;
        }

        @Override
        public void assignMediaItems(
                Optional<String> albumId,
                List<UploadedPhoto> items,
                Consumer<List<Path>> itemsAssigned)
        {
            System.out.println("Create media entries in album: " + albumId);
            items.forEach(upload ->
                    System.out.println(
                            "\t" + upload.uploadToken + " -> " + upload.file));
            itemsAssigned.accept(
                    items
                            .stream()
                            .map(i -> i.file)
                            .collect(Collectors.toList()));
        }
    }

    public static final class Configuration
    {
        final Path rootDirectory;
        final List<Pattern> directoryIncludePatterns;
        final List<Pattern> excludePatterns;
        final boolean createAlbums;

        public Configuration(
                Path rootDirectory,
                List<Pattern> directoryIncludePatterns,
                List<Pattern> excludePatterns,
                boolean createAlbums) {
            this.rootDirectory = rootDirectory;
            this.directoryIncludePatterns = directoryIncludePatterns;
            this.excludePatterns = excludePatterns;
            this.createAlbums = createAlbums;
        }
    }

    public static void push(
            final Configuration configuration,
            final Client client,
            final Database database
    ) throws IOException {
        final Path root = configuration.rootDirectory;

        // Walk the tree, upload as we find things
        Files.walkFileTree(root, new FileVisitor<>() {
            @Override
            public FileVisitResult preVisitDirectory(
                    Path path,
                    BasicFileAttributes basicFileAttributes) {
                if (path.equals(root)) {
                    return FileVisitResult.CONTINUE;
                } else if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) {
                    String directoryAsString = root.relativize(path).toString();
                    if (anyIncludeMatches(directoryAsString)
                            && noExcludeMatches(directoryAsString))
                    {
                        return FileVisitResult.CONTINUE;
                    } else {
                        return FileVisitResult.SKIP_SUBTREE;
                    }
                }

                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(
                    Path path,
                    BasicFileAttributes basicFileAttributes) {
                final Path relativize = root.relativize(path);
                final Path album = relativize.getParent();

                // Album is null if we find files in the root directory.
                // I'd fix that if this were anything other than a sketch!
                if (album != null && shouldUpload(path)) {
                    // Here we upload synchronously, but there's no reason
                    // we couldn't queue the events and have workers call
                    // client in parallel.
                    client.upload(
                            path,
                            album.toString(),
                            database::itemUploaded);
                }

                return FileVisitResult.CONTINUE;
            }

            private boolean noExcludeMatches(String directoryAsString) {
                return configuration
                        .excludePatterns
                        .stream()
                        .noneMatch(p -> p.matcher(directoryAsString).matches());
            }

            private boolean anyIncludeMatches(String directoryAsString) {
                return configuration
                        .directoryIncludePatterns
                        .stream()
                        .anyMatch(p -> p.matcher(directoryAsString).matches());
            }

            private boolean shouldUpload(Path path) {
                // This should check that path is an image or movie file
                return database.requiresUpload(path);
            }

            @Override
            public FileVisitResult visitFileFailed(Path path, IOException e) {
                System.out.println("Visit file failed for " + path);
                e.printStackTrace();
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path path, IOException e) {
                return FileVisitResult.CONTINUE;
            }
        });

        if (configuration.createAlbums) {
            Set<Map.Entry<String, List<UploadedPhoto>>> groupedByAlbumTitle =
                    database.requiringMediaItemCreation()
                            .collect(Collectors.groupingBy(item -> item.albumTitle))
                            .entrySet();
            for (Map.Entry<String, List<UploadedPhoto>> entry :
                    groupedByAlbumTitle) {
                final String albumName = entry.getKey();
                final String albumId = client.ensureAlbumExists(
                        albumName, database::albumCreated);
                client.assignMediaItems(
                        Optional.of(albumId),
                        // We could sort the items here - the current CLI
                        // adds them in an indeterminate order due to threading
                        entry.getValue(),
                        database::batchItemsCreated);
            }
        } else {
            List<UploadedPhoto> uploadedPhotos =
                    database
                            .requiringMediaItemCreation()
                            .collect(Collectors.toList());
            client.assignMediaItems(
                    Optional.empty(),
                    uploadedPhotos,
                    database::batchItemsCreated
            );
        }
    }

    public static void main(String[] args) throws IOException {
        InMemoryDatabase database = new InMemoryDatabase();
        File root = new File("/media/ds212/photo");
        DryRunClient client = new DryRunClient();
        Configuration configuration = new Configuration(
                root.toPath(),
                Arrays.asList(
                        Pattern.compile("2020.*"), Pattern.compile("2019.*")
                ),
                Arrays.asList(
                        Pattern.compile(".*eaDir.*"),
                        Pattern.compile(".*\\.DS_Store.*")
                ),
                true
        );

        push(configuration, client, database);

        System.out.println("And now, push again - nothing should happen");

        push(configuration, client, database);
    }
}