Skip to content

Instantly share code, notes, and snippets.

@lenborje
Created June 2, 2016 12:19
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save lenborje/6d2f92430abe4ba881e3c5ff83736923 to your computer and use it in GitHub Desktop.
Save lenborje/6d2f92430abe4ba881e3c5ff83736923 to your computer and use it in GitHub Desktop.
Minimal zip utility in java. It can process entries in parallel. Utilizes Java 8 parallel Streams combined with the ZIP FileSystem introduced in Java 7.
import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.util.*;
import java.util.stream.*;
import static java.util.stream.Collectors.*;
/**
* This class creates zip archives. Instead of directly using {@link java.util.zip.ZipOutputStream},
* this implementation uses the jar {@link FileSystem} available since Java 1.7.<p>
* The advantage of using a {@code FileSystem} is that it can easily be processed in parallel.<p>
* This class can create zip archives with parallel execution by combining parallel {@link Stream} processing
* with the jar {@code FileSystem}.<p>
* This class has a {@link #main(String[])} method which emulates a minimal command-line zip utility, i.e.
* it can be used to create standard zip archives.
*
* @author Lennart Börjeson
*
*/
public class Zip implements Closeable {
private final FileSystem zipArchive;
private final boolean recursive;
private final boolean parallel;
private final Set<Zip.Options> options;
/**
* Creates and initialises a Zip archive.
* @param archiveName name (file path) of the archive
* @param options {@link Options}
* @throws IOException Thrown on any underlying IO errors
* @throws URISyntaxException Thrown on file name syntax errors.
*/
public Zip(final String archiveName, Options... options) throws IOException, URISyntaxException {
this.options = Collections.unmodifiableSet(Stream.of(options).collect(toSet()));
this.recursive = this.options.contains(Options.RECURSIVE);
this.parallel = this.options.contains(Options.PARALLEL);
final Path zipPath = Paths.get(archiveName);
final Map<String, String> zipParams = new HashMap<>();
zipParams.put("create", "true");
final URI resolvedFileURI = zipPath.toAbsolutePath().toUri();
final URI zipURI = new URI("jar:file", resolvedFileURI.getPath(), (String) null);
System.out.printf("Working on ZIP FileSystem %s, using the options %s%n", zipURI, this.options);
zipArchive = FileSystems.newFileSystem(zipURI, zipParams);
}
/**
* Adds one file to the archive.
*
* @param f
* Path of file to add, not null
*/
public void zipOneFile(final Path f) {
try {
final Path parent = f.getParent();
if (parent != null && parent.getNameCount() > 0)
Files.createDirectories(zipArchive.getPath(parent.toString()));
final Path zipEntryPath = zipArchive.getPath(f.toString());
String message = " adding: %s";
if (Files.exists(zipEntryPath)) {
Files.deleteIfExists(zipEntryPath);
message = " updating: %s";
}
final StringBuilder logbuf = new StringBuilder();
try (OutputStream out = Files.newOutputStream(zipEntryPath)) {
logbuf.append(String.format(message, f));
Files.copy(f, out);
out.flush();
} catch (Exception e) {
System.err.printf("Error adding %s:%n", f);
e.printStackTrace(System.err);
return;
}
final long size = (long) Files.getAttribute(zipEntryPath, "zip:size");
final long compressedSize = (long) Files.getAttribute(zipEntryPath, "zip:compressedSize");
final double compression = (size-compressedSize)*100.0/size;
final int method = (int) Files.getAttribute(zipEntryPath, "zip:method");
final String methodName = method==0?"stored":method<8?"compressed":"deflated";
logbuf.append(String.format(" (%4$s %3$.0f%%)", size, compressedSize, compression, methodName));
System.out.println(logbuf);
} catch (Exception e1) {
throw new RuntimeException(String.format(" Error accessing zip archive for %s:", f), e1);
}
}
@Override
public void close() throws IOException {
zipArchive.close();
}
/**
* Adds files, given as a {@link List} of file names, to this Zip archive.
* <p>
* If the option {@link Options#RECURSIVE} was specified in the constructor,
* any directories specified will be traversed and all files found will be added.
* <p>
* If the option {@link Options#PARALLEL} was specified in the constructor,
* all files found will be added in parallel.
* @param fileNameArgs List of file names, not null
*/
public void addFiles(final List<String> fileNameArgs) {
final List<Path> expandedPaths =
fileNameArgs.stream() // Process file name list
.map(File::new) // String -> File
.flatMap(this::filesWalk) // Find file, or, if recursive, files
.map(Path::normalize) // Ensure no contrived paths
.collect(toList()); // Collect into List! NB! Necessary!
// Do NOT remove the collection into a List!
// Doing so can defeat the desired parallelism.
// By first resolving all directory traversals,
// we ensure all files will be processed in parallel in the next step.
// (This is because the directory traversal parallelises
// badly, whereas the contents of a list does eminently so.)
// If parallel processing requested, use parallel stream,
// else use normal stream.
final Stream<Path> streamOfPaths =
parallel ? expandedPaths.parallelStream() : expandedPaths.stream();
streamOfPaths.forEach(this::zipOneFile); // zip them all!
}
/**
* If the given {@link File} argument represents a real file (i.e.
* {@link File#isFile()} returns {@code true}), converts the given file
* argument to a {@link Stream} of a single {@link Path} (of the given file
* argument).
* <p>
* Else, if {@link Options#RECURSIVE} was specified in the constructor
* {@link Zip#Zip(String, Options...)}, assumes the file represents a directory and then uses
* {@link Files#walk(Path, java.nio.file.FileVisitOption...)} to return a
* {@code Stream} of all real files contained within this directory tree.
* <p>
* Returns an empty stream if any errors are encountered.
*
* @param f
* File, representing a file or directory.
* @return Stream of all Paths resolved
*/
private Stream<Path> filesWalk(final File f) {
// If argument is a file, return directly as single-item stream
if (f.isFile()) {
return Stream.of(f.toPath());
}
// Check if argument is a directory and RECURSIVE option specified
if (f.isDirectory() && this.recursive)
try {
// Traverse directory and return all files found
return Files.walk(f.toPath(), FileVisitOption.FOLLOW_LINKS)
.filter(p -> p.toFile().isFile()); // Only return real files
} catch (IOException e) {
throw new RuntimeException(String.format("Error traversing directory %s", f), e);
}
// Argument is neither file nor directory: Return empty stream
return Stream.empty();
}
/**
* Represents Zip processing options. (Internal to this application; not needed by the jar FileSystem.)
* @author Lennart Börjeson
*
*/
public enum Options {
/**
* Requests that all file additions should be executed in parallel.
*/
PARALLEL,
/**
* Requests that any directory specified as input should be
* recursively traversed and all files found added individually to the
* Zip archive. Paths will be preserved.
*/
RECURSIVE;
/**
* Maps names to options
*/
private static final Map<String, Options>name2option = new HashMap<>();
public final String shortName;
public final String longName;
private static final void register(final Options o) {
name2option.put(o.shortName, o);
name2option.put(o.longName, o);
}
/**
* Collect and register all options
*/
static {
for (Options o : values()) {
register(o);
}
}
/**
* Creates an Options with the given short and long names. Don't specify any hyphen/dash in the name!
* @param shortName Short name
* @param longName Long name, without any leading dashes or hyphens
*/
private Options(final char shortName, final String longName) {
this.shortName = "-"+shortName;
this.longName = "--"+longName;
}
/**
* Creates an Options with the given long name. The short name will be the first character of the long name.
* @param longName Long name, without any leading dashes or hyphens
*/
private Options(final String longName) {
this(longName.charAt(0), longName);
}
/**
* Creates an Options, using the lower case of the declared name as the long name. The short name will be the first character of the long name.
*/
private Options() {
this.longName = "--"+name().toLowerCase();
this.shortName = "-"+name().toLowerCase().charAt(0);
}
/**
* Parses a string as either the long or short representation of an Option.
* @param optionName
* @return Parsed Option
* @throws IllegalOptionException If string argument isn't recognised as an option.
*/
public static Options parseOptionName(final String optionName) {
Options result = name2option.get(optionName);
if (result == null) {
throw new Zip.IllegalOptionException(String.format("Unrecognised option '%s'", optionName));
}
return result;
}
/**
* Prepend a single char (specified as int, as {@link String#chars()} returns integers) with
* a dash, and return this string.
* @param c char, specified as int
* @return "-" + given char
*/
private static String singleCharToOptionName(final int c) {
StringBuilder sb = new StringBuilder("-");
sb.append((char)c);
return sb.toString();
}
/**
* Explode a given single-dash option sequence (e.g. "-rp") to a stream of
* corresponding stream of single-character options (e.g. "-r", "-p").
* @param optionName
* @return Stream of single-character optionNames
*/
public static Stream<String> explodeSingleDashOptions(final String optionName) {
if (optionName.startsWith("--"))
return Stream.of(optionName); // Do nothing, just return this option name
// Else explode concatenated single-character options to individual options
return optionName.substring(1) // Remove first dash
.chars() // explode to single char stream
.mapToObj(Options::singleCharToOptionName); // Convert to option name
}
/**
* Returns a syntax string for this Options, e.g. "[-p|--parallel]"
* @return Single option syntax string
*/
private String syntax() {
return String.format("[%s|%s]", shortName, longName);
}
/**
* Returns the combined syntax string for all Options.
* @return Options syntax string
*/
public static String optionsSyntax() {
return
Stream.of(values())
.map(Options::syntax)
.collect(joining(" "));
}
}
/**
* Thrown when {@link Options#parseOptionName(String)} can't recognise the given argument.
*
* @author Lennart Börjeson
*
*/
public static class IllegalOptionException extends IllegalArgumentException {
private static final long serialVersionUID = 1L;
/**
* Creates this exception.
* @param s Reason message
*/
public IllegalOptionException(final String s) {
super(s);
}
}
/**
* Thrown by {@link Main#main(String[])} when not enough arguments were given.
*
* @author Lennart Börjeson
*
*/
public static class NotEnoughArgumentsException extends RuntimeException {
private static final long serialVersionUID = 1L;
/**
* Creates this exception with the given reason string.
* @param string Reason message
*/
public NotEnoughArgumentsException(final String string) {
super(string);
}
}
/**
* Prints simple usage info.
*/
private static void usage() {
System.err.println();
System.err.printf("Usage: zip %s <zip-archive> <file|dir...>%n", Zip.Options.optionsSyntax());
System.err.println();
System.err.println("");
}
/**
* Main entry point, when launched as stand-alone application.
* @param args Command-line arguments.
*/
public static void main(final String[] args) {
try {
// This list will eventually contain only the file name arguments
final LinkedList<String> fileArgs = Stream.of(args).collect(toCollection(LinkedList::new));
// Collect all option arguments
final List<String> optionArgs = fileArgs.stream().filter(s->s.startsWith("-")).collect(toList());
// Remove option arguments from file name list
fileArgs.removeAll(optionArgs);
// Parse option arguments and convert to options array. Exceptions might be thrown here.
final Zip.Options[] options =
optionArgs.stream()
.flatMap(Zip.Options::explodeSingleDashOptions)
.map(Zip.Options::parseOptionName)
.toArray(Zip.Options[]::new);
// Check argument count. At least one zip file and one file/dir to be added to the zip is required.
if (fileArgs.size()<2) {
throw new NotEnoughArgumentsException("Not enough file name arguments!");
}
final String zipName = fileArgs.removeFirst(); // Remove zip name argument
try (Zip zip = new Zip(zipName, options)) { // Initialise zip archive
zip.addFiles(fileArgs); // Add files
System.out.print("completed zip archive, now closing... ");
}
System.out.println("done!");
} catch (NotEnoughArgumentsException | IllegalOptionException re) {
System.err.println(re.getMessage());
usage();
System.exit(1);
} catch (Exception e) {
e.printStackTrace(System.err);
System.exit(2);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment