From f76a3d2d6250cca55cc568683dc724343187c466 Mon Sep 17 00:00:00 2001 From: comp500 Date: Sat, 1 Jun 2019 12:33:28 +0100 Subject: [PATCH] Implement Zip request handler --- .../installer/request/HandlerManager.java | 131 +++--------------- .../installer/request/IRequestHandler.java | 11 +- .../handlers/RequestHandlerGithub.java | 25 ++++ .../request/handlers/RequestHandlerHTTP.java | 12 +- .../request/handlers/RequestHandlerZip.java | 114 +++++++++++++++ .../installer/util/CachedInputStream.java | 25 ---- 6 files changed, 181 insertions(+), 137 deletions(-) create mode 100644 src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java create mode 100644 src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java delete mode 100644 src/main/java/link/infra/packwiz/installer/util/CachedInputStream.java diff --git a/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java b/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java index fea00c4..4112522 100644 --- a/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java +++ b/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java @@ -3,16 +3,9 @@ package link.infra.packwiz.installer.request; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.Consumer; +import link.infra.packwiz.installer.request.handlers.RequestHandlerGithub; import link.infra.packwiz.installer.request.handlers.RequestHandlerHTTP; public abstract class HandlerManager { @@ -20,6 +13,7 @@ public abstract class HandlerManager { public static List handlers = new ArrayList(); static { + handlers.add(new RequestHandlerGithub()); handlers.add(new RequestHandlerHTTP()); } @@ -37,7 +31,7 @@ public abstract class HandlerManager { return loc; } - public static InputStream getFileInputStream(URI loc) { + public static InputStream getFileInputStream(URI loc) throws Exception { for (IRequestHandler handler : handlers) { if (handler.matchesHandler(loc)) { return handler.getFileInputStream(loc); @@ -46,108 +40,29 @@ public abstract class HandlerManager { return null; } - public class RequestTaskManager { - private Map tasks = new HashMap(); - private int numTasks = 0; - private int numRemainingTasks = 0; - private ExecutorService threadPool = Executors.newFixedThreadPool(10); - - public int getNumTasks() { - synchronized (tasks) { - return numTasks; - } - } - - public int getNumTasksRemaining() { - synchronized (tasks) { - return numRemainingTasks; - } - } - - // UHHHHHHHH fix this maybe? it's O(n^2) and kinda bad - // because it has to be done every time you download a thing, to check if you can download more - // maybe: - // store list of dependents and dependencies for all tasks, update when task completes?????? - // this is hard!!! - // still have the issue of how to read while also allowing it to be re-read (cachedinputstream) - // or go to a dependency free system in some way, but just have 2 types of request: progress only, and actual download??????? - // and how to multithread it??? - private boolean hasRemainingDependencies(RequestTask task) { - synchronized (tasks) { - return Arrays.stream(task.getDependencies()).filter(depUri -> { - RequestTask depTask = tasks.get(depUri); - if (depTask == null) { - return true; - } else { - return !depTask.isDone(); - } - }).count() > 0; - } - } - - public void enqueue(URI loc) { - // get a requesttask somehow - RequestTask task = null; - - Stack toEnqueue = new Stack(); - - URI[] remainingDeps; - synchronized (tasks) { - remainingDeps = Arrays.stream(task.getDependencies()).filter(depUri -> { - RequestTask depTask = tasks.get(depUri); - if (depTask == null) { - toEnqueue.add(depUri); - return true; - } else { - return !depTask.isDone(); - } - }).toArray(URI[]::new); - } - - synchronized (tasks) { - tasks.put(loc, null); - } - - while (toEnqueue.size() > 0) { - enqueue(toEnqueue.pop()); - } - - if (remainingDeps.length == 0) { - // execute it - // after executing, check deps of other tasks - } - } - } - - public abstract class RequestTask implements Future { - - protected final Consumer setProgress; - public final URI requestLocation; - - public abstract URI[] getDependencies(); - - public RequestTask(Consumer setProgress, URI requestLocation) { - this.setProgress = setProgress; - this.requestLocation = requestLocation; - } - } - - // how to handle progress of zip download, for zip/github downloads? + // To enqueue stuff: +// private ExecutorService threadPool = Executors.newFixedThreadPool(10); +// CompletionService completionService = new ExecutorCompletionService(threadPool); +// +// public Future enqueue(URI loc) { +// for (IRequestHandler handler : handlers) { +// if (handler.matchesHandler(loc)) { +// return completionService.submit(new Callable() { +// public InputStream call() { +// return handler.getFileInputStream(loc); +// } +// }); +// } +// } +// // TODO: throw error?? +// return null; +// } + // Use completionService.take() to get (waits until available) a Future, where you can call .get() and handle exceptions etc // github toml resolution // e.g. https://github.com/comp500/Demagnetize -> demagnetize.toml // https://github.com/comp500/Demagnetize/blob/master/demagnetize.toml - // Use a Request class? - // sub requests, can get progress (but not data) of sub things - // function to get length -> -1 means indeterminate - // function / callback to get progress - // input stream progress tracker, like the bootstrapper? - // https://docs.oracle.com/javase/tutorial/uiswing/components/progress.html -> swingworker, other magic to get progress - - // stack of request tasks - // deduplicated - - - // UHHHHH I THINK I HAVE THE HALTING PROBLEM AND ITS NOT NICE + // To handle "progress", just count tasks, rather than individual progress + // It'll look bad, especially for zip-based things, but it should work fine } diff --git a/src/main/java/link/infra/packwiz/installer/request/IRequestHandler.java b/src/main/java/link/infra/packwiz/installer/request/IRequestHandler.java index d3f8bbb..e2896c3 100644 --- a/src/main/java/link/infra/packwiz/installer/request/IRequestHandler.java +++ b/src/main/java/link/infra/packwiz/installer/request/IRequestHandler.java @@ -3,6 +3,9 @@ package link.infra.packwiz.installer.request; import java.io.InputStream; import java.net.URI; +/** + * IRequestHandler handles requests for locations specified in modpack metadata. + */ public interface IRequestHandler { public boolean matchesHandler(URI loc); @@ -11,6 +14,12 @@ public interface IRequestHandler { return loc; } - public InputStream getFileInputStream(URI loc); + /** + * Gets the InputStream for a location. Must be threadsafe. + * @param loc The location to be read + * @return The InputStream containing the data of the file + * @throws Exception + */ + public InputStream getFileInputStream(URI loc) throws Exception; } diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java new file mode 100644 index 0000000..009b828 --- /dev/null +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java @@ -0,0 +1,25 @@ +package link.infra.packwiz.installer.request.handlers; + +import java.net.URI; + +public class RequestHandlerGithub extends RequestHandlerZip { + + @Override + protected URI getZipUri(URI loc) throws Exception { + // TODO Auto-generated method stub + return null; + } + + @Override + protected URI getLocationInZip(URI loc) throws Exception { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean matchesHandler(URI loc) { + // TODO Auto-generated method stub + return false; + } + +} diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java index dc8ac64..0eb2263 100644 --- a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java @@ -1,7 +1,10 @@ package link.infra.packwiz.installer.request.handlers; +import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URLConnection; import link.infra.packwiz.installer.request.IRequestHandler; @@ -14,9 +17,12 @@ public class RequestHandlerHTTP implements IRequestHandler { } @Override - public InputStream getFileInputStream(URI loc) { - // TODO Auto-generated method stub - return null; + public InputStream getFileInputStream(URI loc) throws Exception { + URLConnection conn = loc.toURL().openConnection(); + conn.addRequestProperty("Accept", "application/octet-stream"); + // 30 second read timeout + conn.setReadTimeout(30 * 1000); + return conn.getInputStream(); } } diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java new file mode 100644 index 0000000..36c94df --- /dev/null +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java @@ -0,0 +1,114 @@ +package link.infra.packwiz.installer.request.handlers; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public abstract class RequestHandlerZip extends RequestHandlerHTTP { + + private class ZipReader { + + private final ZipInputStream zis; + private final Map readFiles = new HashMap(); + // Write lock implies access to ZipInputStream - only 1 thread must read at a time! + final ReentrantReadWriteLock filesLock = new ReentrantReadWriteLock(); + private ZipEntry entry; + + public ZipReader(InputStream zip) { + zis = new ZipInputStream(zip); + } + + // File write lock must be obtained before calling this function + private byte[] readCurrFile() throws IOException { + byte[] bytes = new byte[(int) entry.getSize()]; + DataInputStream dis = new DataInputStream(zis); + dis.readFully(bytes); + return bytes; + } + + // File write lock must be obtained before calling this function + private byte[] findFile(URI loc) throws IOException, URISyntaxException { + while (true) { + entry = zis.getNextEntry(); + if (entry == null) { + return null; + } + byte[] data = readCurrFile(); + if (loc.equals(new URI(entry.getName()))) { + return data; + } else { + readFiles.put(loc, data); + } + } + } + + public InputStream getFileInputStream(URI loc) throws Exception { + filesLock.readLock().lock(); + byte[] file = readFiles.get(loc); + filesLock.readLock().unlock(); + if (file != null) { + // Assume files are only read once, allow GC + filesLock.writeLock().lock(); + readFiles.remove(loc); + filesLock.writeLock().unlock(); + return new ByteArrayInputStream(file); + } + filesLock.writeLock().lock(); + // Test again after receiving write lock + file = readFiles.get(loc); + if (file != null) { + // Assume files are only read once, allow GC + readFiles.remove(loc); + filesLock.writeLock().unlock(); + return new ByteArrayInputStream(file); + } + + file = findFile(loc); + filesLock.writeLock().unlock(); + if (file != null) { + return new ByteArrayInputStream(file); + } + return null; + } + + } + + private final Map cache = new HashMap(); + final ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock(); + + protected abstract URI getZipUri(URI loc) throws Exception; + + protected abstract URI getLocationInZip(URI loc) throws Exception; + + @Override + public abstract boolean matchesHandler(URI loc); + + @Override + public InputStream getFileInputStream(URI loc) throws Exception { + URI zipUri = getZipUri(loc); + cacheLock.readLock().lock(); + ZipReader zr = cache.get(zipUri); + cacheLock.readLock().unlock(); + if (zr == null) { + cacheLock.writeLock().lock(); + // Recheck, because unlocking read lock allows another thread to modify it + zr = cache.get(zipUri); + if (zr == null) { + zr = new ZipReader(super.getFileInputStream(zipUri)); + cache.put(zipUri, zr); + } + cacheLock.writeLock().unlock(); + } + + return zr.getFileInputStream(getLocationInZip(loc)); + } + +} diff --git a/src/main/java/link/infra/packwiz/installer/util/CachedInputStream.java b/src/main/java/link/infra/packwiz/installer/util/CachedInputStream.java deleted file mode 100644 index 283e8ea..0000000 --- a/src/main/java/link/infra/packwiz/installer/util/CachedInputStream.java +++ /dev/null @@ -1,25 +0,0 @@ -package link.infra.packwiz.installer.util; - -import java.io.FilterInputStream; -import java.io.InputStream; - -public class CachedInputStream extends FilterInputStream { - - public CachedInputStream(InputStream stream, boolean isMaster) { - if (!isMaster) { - - } - super(stream); - } - - public CachedInputStream(InputStream stream) { - super(stream, true); - } - - public CachedInputStream getNewStream() { - - } - - - -}