Implement Zip request handler

src/main/java/link/infra/packwiz/installer/request/HandlerManager.java

@@ -3,16 +3,9 @@ package link.infra.packwiz.installer.request;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Stack;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.function.Consumer;
 
+import link.infra.packwiz.installer.request.handlers.RequestHandlerGithub;
 import link.infra.packwiz.installer.request.handlers.RequestHandlerHTTP;
 
 public abstract class HandlerManager {
@@ -20,6 +13,7 @@ public abstract class HandlerManager {
 	public static List<IRequestHandler> handlers = new ArrayList<IRequestHandler>();
 	
 	static {
+		handlers.add(new RequestHandlerGithub());
 		handlers.add(new RequestHandlerHTTP());
 	}
 	
@@ -37,7 +31,7 @@ public abstract class HandlerManager {
 		return loc;
 	}
 
-	public static InputStream getFileInputStream(URI loc) {
+	public static InputStream getFileInputStream(URI loc) throws Exception {
 		for (IRequestHandler handler : handlers) {
 			if (handler.matchesHandler(loc)) {
 				return handler.getFileInputStream(loc);
@@ -46,108 +40,29 @@ public abstract class HandlerManager {
 		return null;
 	}
 	
-	public class RequestTaskManager {
+	// To enqueue stuff:
-		private Map<URI, RequestTask> tasks = new HashMap<URI, RequestTask>();
+//	private ExecutorService threadPool = Executors.newFixedThreadPool(10);
-		private int numTasks = 0;
+//	CompletionService<InputStream> completionService = new ExecutorCompletionService<InputStream>(threadPool);
-		private int numRemainingTasks = 0;
+//
-		private ExecutorService threadPool = Executors.newFixedThreadPool(10);
+//	public Future<InputStream> enqueue(URI loc) {
-		
+//		for (IRequestHandler handler : handlers) {
-		public int getNumTasks() {
+//			if (handler.matchesHandler(loc)) {
-			synchronized (tasks) {
+//				return completionService.submit(new Callable<InputStream>() {
-				return numTasks;
+//					public InputStream call() {
-			}
+//						return handler.getFileInputStream(loc);
-		}
+//					}
-		
+//				});
-		public int getNumTasksRemaining() {
+//			}
-			synchronized (tasks) {
+//		}
-				return numRemainingTasks;
+//		// TODO: throw error??
-			}
+//		return null;
-		}
+//	}
-		
+	// Use completionService.take() to get (waits until available) a Future<InputStream>, where you can call .get() and handle exceptions etc
-		// UHHHHHHHH fix this maybe? it's O(n^2) and kinda bad
-		// because it has to be done every time you download a thing, to check if you can download more
-		// maybe:
-		// store list of dependents and dependencies for all tasks, update when task completes??????
-		// this is hard!!!
-		// still have the issue of how to read while also allowing it to be re-read (cachedinputstream)
-		// or go to a dependency free system in some way, but just have 2 types of request: progress only, and actual download???????
-		// and how to multithread it???
-		private boolean hasRemainingDependencies(RequestTask task) {
-			synchronized (tasks) {
-				return Arrays.stream(task.getDependencies()).filter(depUri -> {
-					RequestTask depTask = tasks.get(depUri);
-					if (depTask == null) {
-						return true;
-					} else {
-						return !depTask.isDone();
-					}
-				}).count() > 0;
-			}
-		}
-		
-		public void enqueue(URI loc) {
-			// get a requesttask somehow
-			RequestTask task = null;
-			
-			Stack<URI> toEnqueue = new Stack<URI>();
-			
-			URI[] remainingDeps;
-			synchronized (tasks) {
-				remainingDeps = Arrays.stream(task.getDependencies()).filter(depUri -> {
-					RequestTask depTask = tasks.get(depUri);
-					if (depTask == null) {
-						toEnqueue.add(depUri);
-						return true;
-					} else {
-						return !depTask.isDone();
-					}
-				}).toArray(URI[]::new);
-			}
-			
-			synchronized (tasks) {
-				tasks.put(loc, null);
-			}
-			
-			while (toEnqueue.size() > 0) {
-				enqueue(toEnqueue.pop());
-			}
-			
-			if (remainingDeps.length == 0) {
-				// execute it
-				// after executing, check deps of other tasks
-			}
-		}
-	}
-	
-	public abstract class RequestTask implements Future<byte[]> {
-		
-		protected final Consumer<Integer> setProgress;
-		public final URI requestLocation;
-		
-		public abstract URI[] getDependencies();
-		
-		public RequestTask(Consumer<Integer> setProgress, URI requestLocation) {
-			this.setProgress = setProgress;
-			this.requestLocation = requestLocation;
-		}
-	}
-	
-	// how to handle progress of zip download, for zip/github downloads?
 	
 	// github toml resolution
 	// e.g. https://github.com/comp500/Demagnetize -> demagnetize.toml
 	// https://github.com/comp500/Demagnetize/blob/master/demagnetize.toml
 	
-	// Use a Request class?
+	// To handle "progress", just count tasks, rather than individual progress
-	// sub requests, can get progress (but not data) of sub things
+	// It'll look bad, especially for zip-based things, but it should work fine
-	// function to get length -> -1 means indeterminate
-	// function / callback to get progress
-	// input stream progress tracker, like the bootstrapper?
-	// https://docs.oracle.com/javase/tutorial/uiswing/components/progress.html -> swingworker, other magic to get progress
-	
-	// stack of request tasks
-	// deduplicated
-	
-	
-	// UHHHHH I THINK I HAVE THE HALTING PROBLEM AND ITS NOT NICE
 }

src/main/java/link/infra/packwiz/installer/request/IRequestHandler.java

@@ -3,6 +3,9 @@ package link.infra.packwiz.installer.request;
 import java.io.InputStream;
 import java.net.URI;
 
+/**
+ * IRequestHandler handles requests for locations specified in modpack metadata.
+ */
 public interface IRequestHandler {
 	
 	public boolean matchesHandler(URI loc);
@@ -11,6 +14,12 @@ public interface IRequestHandler {
 		return loc;
 	}
 	
-	public InputStream getFileInputStream(URI loc);
+	/**
+	 * Gets the InputStream for a location. Must be threadsafe.
+	 * @param loc The location to be read
+	 * @return The InputStream containing the data of the file
+	 * @throws Exception
+	 */
+	public InputStream getFileInputStream(URI loc) throws Exception;
 
 }

src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java

@@ -0,0 +1,25 @@
+package link.infra.packwiz.installer.request.handlers;
+
+import java.net.URI;
+
+public class RequestHandlerGithub extends RequestHandlerZip {
+
+	@Override
+	protected URI getZipUri(URI loc) throws Exception {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	protected URI getLocationInZip(URI loc) throws Exception {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean matchesHandler(URI loc) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+}

src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java

@@ -1,7 +1,10 @@
 package link.infra.packwiz.installer.request.handlers;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URI;
+import java.net.URLConnection;
 
 import link.infra.packwiz.installer.request.IRequestHandler;
 
@@ -14,9 +17,12 @@ public class RequestHandlerHTTP implements IRequestHandler {
 	}
 
 	@Override
-	public InputStream getFileInputStream(URI loc) {
+	public InputStream getFileInputStream(URI loc) throws Exception {
-		// TODO Auto-generated method stub
+		URLConnection conn = loc.toURL().openConnection();
-		return null;
+		conn.addRequestProperty("Accept", "application/octet-stream");
+		// 30 second read timeout
+		conn.setReadTimeout(30 * 1000);
+		return conn.getInputStream();
 	}
 
 }

src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java

@@ -0,0 +1,114 @@
+package link.infra.packwiz.installer.request.handlers;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public abstract class RequestHandlerZip extends RequestHandlerHTTP {
+	
+	private class ZipReader {
+		
+		private final ZipInputStream zis;
+		private final Map<URI, byte[]> readFiles = new HashMap<URI, byte[]>();
+		// Write lock implies access to ZipInputStream - only 1 thread must read at a time!
+		final ReentrantReadWriteLock filesLock = new ReentrantReadWriteLock();
+		private ZipEntry entry;
+
+		public ZipReader(InputStream zip) {
+			zis = new ZipInputStream(zip);
+		}
+		
+		// File write lock must be obtained before calling this function
+		private byte[] readCurrFile() throws IOException {
+			byte[] bytes = new byte[(int) entry.getSize()];
+			DataInputStream dis = new DataInputStream(zis);
+			dis.readFully(bytes);
+			return bytes;
+		}
+		
+		// File write lock must be obtained before calling this function
+		private byte[] findFile(URI loc) throws IOException, URISyntaxException {
+			while (true) {
+				entry = zis.getNextEntry();
+				if (entry == null) {
+					return null;
+				}
+				byte[] data = readCurrFile();
+				if (loc.equals(new URI(entry.getName()))) {
+					return data;
+				} else {
+					readFiles.put(loc, data);
+				}
+			}
+		}
+		
+		public InputStream getFileInputStream(URI loc) throws Exception {
+			filesLock.readLock().lock();
+			byte[] file = readFiles.get(loc);
+			filesLock.readLock().unlock();
+			if (file != null) {
+				// Assume files are only read once, allow GC
+				filesLock.writeLock().lock();
+				readFiles.remove(loc);
+				filesLock.writeLock().unlock();
+				return new ByteArrayInputStream(file);
+			}
+			filesLock.writeLock().lock();
+			// Test again after receiving write lock
+			file = readFiles.get(loc);
+			if (file != null) {
+				// Assume files are only read once, allow GC
+				readFiles.remove(loc);
+				filesLock.writeLock().unlock();
+				return new ByteArrayInputStream(file);
+			}
+			
+			file = findFile(loc);
+			filesLock.writeLock().unlock();
+			if (file != null) {
+				return new ByteArrayInputStream(file);
+			}
+			return null;
+		}
+		
+	}
+	
+	private final Map<URI, ZipReader> cache = new HashMap<URI, ZipReader>();
+	final ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock();
+	
+	protected abstract URI getZipUri(URI loc) throws Exception;
+	
+	protected abstract URI getLocationInZip(URI loc) throws Exception;
+	
+	@Override
+	public abstract boolean matchesHandler(URI loc);
+
+	@Override
+	public InputStream getFileInputStream(URI loc) throws Exception {
+		URI zipUri = getZipUri(loc);
+		cacheLock.readLock().lock();
+		ZipReader zr = cache.get(zipUri);
+		cacheLock.readLock().unlock();
+		if (zr == null) {
+			cacheLock.writeLock().lock();
+			// Recheck, because unlocking read lock allows another thread to modify it
+			zr = cache.get(zipUri);
+			if (zr == null) {
+				zr = new ZipReader(super.getFileInputStream(zipUri));
+				cache.put(zipUri, zr);
+			}
+			cacheLock.writeLock().unlock();
+		}
+		
+		return zr.getFileInputStream(getLocationInZip(loc));
+	}
+
+}

src/main/java/link/infra/packwiz/installer/util/CachedInputStream.java

@@ -1,25 +0,0 @@
-package link.infra.packwiz.installer.util;
-
-import java.io.FilterInputStream;
-import java.io.InputStream;
-
-public class CachedInputStream extends FilterInputStream {
-	
-	public CachedInputStream(InputStream stream, boolean isMaster) {
-		if (!isMaster) {
-			
-		}
-		super(stream);
-	}
-
-	public CachedInputStream(InputStream stream) {
-		super(stream, true);
-	}
-	
-	public CachedInputStream getNewStream() {
-		
-	}
-	
-	
-
-}