Commit WIP downloading process

src/main/java/link/infra/packwiz/installer/UpdateManager.java

@@ -7,7 +7,18 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
 import java.net.URI;
+import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonIOException;
@@ -15,6 +26,7 @@ import com.google.gson.JsonSyntaxException;
 import com.google.gson.annotations.SerializedName;
 import com.moandjiezana.toml.Toml;
 
+import link.infra.packwiz.installer.metadata.IndexFile;
 import link.infra.packwiz.installer.metadata.ManifestFile;
 import link.infra.packwiz.installer.metadata.PackFile;
 import link.infra.packwiz.installer.metadata.hash.Hash;
@@ -35,10 +47,8 @@ public class UpdateManager {
 
 		public static enum Side {
 			@SerializedName("client")
-			CLIENT("client"),
-			@SerializedName("server")
-			SERVER("server"),
-			@SerializedName("both")
+			CLIENT("client"), @SerializedName("server")
+			SERVER("server"), @SerializedName("both")
 			BOTH("both", new Side[] { CLIENT, SERVER });
 
 			private final String sideName;
@@ -138,6 +148,8 @@ public class UpdateManager {
 
 		System.out.println(pf.name);
 
+		processIndex(HandlerManager.getNewLoc(opts.downloadURI, pf.index.file),
+				new Hash(pf.index.hash, pf.index.hashFormat), manifest);
 
 		// When successfully updated
 		manifest.packFileHash = packFileHash;
@@ -149,10 +161,161 @@ public class UpdateManager {
 			// TODO: add message?
 			ui.handleException(e);
 		}
-		
+
 	}
 
 	protected void checkOptions() {
 		// TODO: implement
 	}
+
+	protected void processIndex(URI indexUri, Hash indexHash, ManifestFile manifest) {
+		Hash.HashInputStream indexFileStream;
+		try {
+			InputStream stream = HandlerManager.getFileInputStream(opts.downloadURI);
+			if (stream == null) {
+				throw new Exception("Index file URI is invalid, is it supported?");
+			}
+			indexFileStream = new Hash.HashInputStream(stream, indexHash);
+		} catch (Exception e) {
+			// TODO: still launch the game if updating doesn't work?
+			// TODO: ask user if they want to launch the game, exit(1) if they don't
+			ui.handleExceptionAndExit(e);
+			return;
+		}
+		IndexFile indexFile;
+		try {
+			indexFile = new Toml().read(indexFileStream).to(IndexFile.class);
+		} catch (IllegalStateException e) {
+			ui.handleExceptionAndExit(e);
+			return;
+		}
+
+		if (!indexFileStream.hashIsEqual()) {
+			System.out.println("Hash problems!!!!!!!");
+			// TODO: throw exception
+		}
+
+		// TODO: progress bar
+		ConcurrentLinkedQueue<Exception> exceptionQueue = new ConcurrentLinkedQueue<Exception>();
+		List<IndexFile.File> newFiles = indexFile.files.stream().filter(f -> {
+			ManifestFile.File cachedFile = manifest.cachedFiles.get(f.file);
+			Hash newHash = new Hash(f.hash, f.hashFormat);
+			return cachedFile == null || newHash.equals(cachedFile.hash);
+		}).parallel().map(f -> {
+			try {
+				f.downloadMeta(indexFile, indexUri);
+			} catch (Exception e) {
+				exceptionQueue.add(e);
+			}
+			return f;
+		}).collect(Collectors.toList());
+
+		for (Exception e : exceptionQueue) {
+			// TODO: collect all exceptions, present in one dialog
+			ui.handleException(e);
+		}
+
+		// TODO: present options
+		// TODO: all options should be presented, not just new files!!!!!!!
+		// and options should be readded to newFiles after option -> true
+		newFiles.stream().filter(f -> f.linkedFile != null).filter(f -> f.linkedFile.option != null).map(f -> {
+			return "option: " + (f.linkedFile.option.description == null ? "null" : f.linkedFile.option.description);
+		}).forEachOrdered(desc -> {
+			System.out.println(desc);
+		});
+
+		// TODO: different thread pool type?
+		ExecutorService threadPool = Executors.newFixedThreadPool(10);
+		CompletionService<DownloadCompletion> completionService = new ExecutorCompletionService<DownloadCompletion>(
+				threadPool);
+
+		for (IndexFile.File f : newFiles) {
+			ManifestFile.File cachedFile = manifest.cachedFiles.get(f.file);
+			completionService.submit(new Callable<DownloadCompletion>() {
+				public DownloadCompletion call() {
+					DownloadCompletion dc = new DownloadCompletion();
+					dc.file = f;
+
+					if (cachedFile.linkedFileHash != null && f.linkedFile != null) {
+						try {
+							if (cachedFile.linkedFileHash.equals(f.linkedFile.getHash())) {
+								// Do nothing, the file didn't change
+								// TODO: but if the hash of the metafile changed, what did change?????
+								// should this be checked somehow??
+								return dc;
+							}
+						} catch (Exception e) {}
+					}
+
+					try {
+						InputStream stream = f.getInputStream(indexUri);
+						if (stream == null) {
+							throw new Exception("Failed to open download stream");
+						}
+						Hash.HashInputStream fileStream = new Hash.HashInputStream(stream, f.getHash());
+						// UGHHHHHH if you're reading this point in history
+						// this is the point where i change EVERYTHING to okio because it's 1000000000000 times nicer for this
+						byte[] data = fileStream.readAllBytes();
+						Files.copy(fileStream, Paths.get(opts.packFolder, f.getDestURI().toString()), StandardCopyOption.REPLACE_EXISTING);
+						
+						return dc;
+					} catch (Exception e) {
+						dc.err = e;
+						return dc;
+					}
+				}
+			});
+		}
+
+		for (int i = 0; i < newFiles.size(); i++) {
+			DownloadCompletion ret;
+			try {
+				ret = completionService.take().get();
+			} catch (InterruptedException | ExecutionException e) {
+				// TODO: collect all exceptions, present in one dialog
+				ui.handleException(e);
+				ret = null;
+			}
+			// Update manifest
+			if (ret != null && ret.err == null && ret.file != null) {
+				ManifestFile.File newCachedFile = new ManifestFile.File();
+				try {
+					newCachedFile.hash = ret.file.getHash();
+					if (newCachedFile.hash == null) {
+						throw new Exception("Invalid hash!");
+					}
+				} catch (Exception e) {
+					ret.err = e;
+				}
+				if (ret.file.metafile && ret.file.linkedFile != null) {
+					newCachedFile.isOptional = ret.file.linkedFile.isOptional();
+					if (newCachedFile.isOptional) {
+						newCachedFile.optionValue = ret.file.optionValue;
+					}
+					try {
+						newCachedFile.linkedFileHash = ret.file.linkedFile.getHash();
+					} catch (Exception e) {
+						ret.err = e;
+					}
+				}
+			}
+			// TODO: show errors properly?
+			String progress;
+			if (ret != null && ret.file != null) {
+				progress = "Downloaded " + ret.file.getName();
+			} else if (ret.err != null) {
+				progress = "Failed to download: " + ret.err.getMessage();
+			} else {
+				progress = "Failed to download, unknown reason";
+			}
+			ui.submitProgress(new InstallProgress(progress, i + 1, newFiles.size()));
+		}
+		// option = false file hashes should be stored to disk, but not downloaded
+		// TODO: don't include optional files in progress????
+	}
+
+	private class DownloadCompletion {
+		Exception err;
+		IndexFile.File file;
+	}
 }

src/main/java/link/infra/packwiz/installer/metadata/IndexFile.java

@@ -1,8 +1,15 @@
 package link.infra.packwiz.installer.metadata;
 
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Paths;
 import java.util.List;
 
 import com.google.gson.annotations.SerializedName;
+import com.moandjiezana.toml.Toml;
+
+import link.infra.packwiz.installer.metadata.hash.Hash;
+import link.infra.packwiz.installer.request.HandlerManager;
 
 public class IndexFile {
 	@SerializedName("hash-format")
@@ -10,12 +17,88 @@ public class IndexFile {
 	public List<File> files;
 	
 	public static class File {
-		public String file;
+		public URI file;
 		@SerializedName("hash-format")
 		public String hashFormat;
 		public String hash;
+		// TODO: implement
 		public String alias;
 		public boolean metafile;
+		// TODO: implement
 		public boolean preserve;
+
+		public transient ModFile linkedFile;
+		public transient URI linkedFileURI;
+		public transient boolean optionValue = true;
+
+		public void downloadMeta(IndexFile parentIndexFile, URI indexUri) throws Exception {
+			if (!metafile) {
+				return;
+			}
+			if (hashFormat == null || hashFormat.length() == 0) {
+				hashFormat = parentIndexFile.hashFormat;
+			}
+			Hash fileHash = new Hash(hash, hashFormat);
+			linkedFileURI = HandlerManager.getNewLoc(indexUri, file);
+			InputStream stream = HandlerManager.getFileInputStream(linkedFileURI);
+			if (stream == null) {
+				throw new Exception("Index file URI is invalid, is it supported?");
+			}
+			Hash.HashInputStream fileStream = new Hash.HashInputStream(stream, fileHash);
+
+			linkedFile = new Toml().read(fileStream).to(ModFile.class);
+			if (!fileStream.hashIsEqual()) {
+				throw new Exception("Invalid mod file hash");
+			}
+		}
+
+		public InputStream getInputStream(URI indexUri) throws Exception {
+			if (metafile) {
+				if (linkedFile == null) {
+					throw new Exception("Linked file doesn't exist!");
+				}
+				return linkedFile.getInputStream(linkedFileURI);
+			} else {
+				URI newLoc = HandlerManager.getNewLoc(indexUri, file);
+				if (newLoc == null) {
+					throw new Exception("Index file URI is invalid");
+				}
+				return HandlerManager.getFileInputStream(newLoc);
+			}
+		}
+
+		public Hash getHash() throws Exception {
+			if (hash == null) {
+				throw new Exception("Index file doesn't have a hash");
+			}
+			if (hashFormat == null) {
+				throw new Exception("Index file doesn't have a hash format");
+			}
+			return new Hash(hash, hashFormat);
+		}
+
+		public String getName() {
+			if (metafile) {
+				if (linkedFile != null) {
+					if (linkedFile.name != null) {
+						return linkedFile.name;
+					} else if (linkedFile.filename != null) {
+						return linkedFile.filename;
+					}
+				}
+			}
+			if (file != null) {
+				return Paths.get(file.getPath()).getFileName().toString();
+			}
+			return file.getPath();
+		}
+
+		public URI getDestURI() {
+			if (metafile && linkedFile != null) {
+				return file.resolve(linkedFile.filename);
+			} else {
+				return file;
+			}
+		}
 	}
 }

src/main/java/link/infra/packwiz/installer/metadata/ManifestFile.java

@@ -1,13 +1,20 @@
 package link.infra.packwiz.installer.metadata;
 
+import java.net.URI;
+import java.util.Map;
+
 import link.infra.packwiz.installer.metadata.hash.Hash;
 
 public class ManifestFile {
 	
 	public Hash packFileHash = null;
 	public Hash indexFileHash = null;
+	public Map<URI, File> cachedFiles;
 
 	public static class File {
 		public Hash hash = null;
+		public boolean isOptional = false;
+		public boolean optionValue = true;
+		public Hash linkedFileHash = null;
 	}
 }

src/main/java/link/infra/packwiz/installer/metadata/ModFile.java

@@ -1,13 +1,16 @@
 package link.infra.packwiz.installer.metadata;
 
+import java.io.InputStream;
 import java.net.URI;
 import java.util.Map;
 
 import com.google.gson.annotations.SerializedName;
 
 import link.infra.packwiz.installer.UpdateManager.Options.Side;
+import link.infra.packwiz.installer.metadata.hash.Hash;
+import link.infra.packwiz.installer.request.HandlerManager;
 
-class ModFile {
+public class ModFile {
 	public String name;
 	public String filename;
 	public Side side;
@@ -30,4 +33,39 @@ class ModFile {
 		public boolean defaultValue;
 	}
 
+	public InputStream getInputStream(URI baseLoc) throws Exception {
+		if (download == null) {
+			throw new Exception("Metadata file doesn't have download");
+		}
+		if (download.url == null) {
+			throw new Exception("Metadata file doesn't have a download URI");
+		}
+		URI newLoc = HandlerManager.getNewLoc(baseLoc, download.url);
+		if (newLoc == null) {
+			throw new Exception("Metadata file URI is invalid");
+		}
+		
+		return HandlerManager.getFileInputStream(newLoc);
+	}
+
+	public Hash getHash() throws Exception {
+		if (download == null) {
+			throw new Exception("Metadata file doesn't have download");
+		}
+		if (download.hash == null) {
+			throw new Exception("Metadata file doesn't have a hash");
+		}
+		if (download.hashFormat == null) {
+			throw new Exception("Metadata file doesn't have a hash format");
+		}
+		return new Hash(download.hash, download.hashFormat);
+	}
+
+	public boolean isOptional() {
+		if (option != null) {
+			return option.optional;
+		}
+		return false;
+	}
+
 }

src/main/java/link/infra/packwiz/installer/metadata/PackFile.java

@@ -1,5 +1,6 @@
 package link.infra.packwiz.installer.metadata;
 
+import java.net.URI;
 import java.util.Map;
 
 import com.google.gson.annotations.SerializedName;
@@ -9,7 +10,7 @@ public class PackFile {
 
 	public IndexFileLoc index;
 	public static class IndexFileLoc {
-		public String file;
+		public URI file;
 		@SerializedName("hash-format")
 		public String hashFormat;
 		public String hash;

src/main/java/link/infra/packwiz/installer/metadata/hash/Hash.java

@@ -104,11 +104,11 @@ public class Hash {
 			return output;
 		}
 
-		public boolean isNew() {
-			if (output != null) {
-				return !output.equals(compare);
+		public boolean hashIsEqual() {
+			if (output == null) {
+				get();
 			}
-			return false;
+			return !output.equals(compare);
 		}
 
 	}

src/main/java/link/infra/packwiz/installer/request/HandlerManager.java

@@ -18,10 +18,10 @@ public abstract class HandlerManager {
 	}
 	
 	public static URI getNewLoc(URI base, URI loc) {
+		if (loc == null) return null;
 		if (base != null) {
 			loc = base.resolve(loc);
 		}
-		if (loc == null) return null;
 		
 		for (IRequestHandler handler : handlers) {
 			if (handler.matchesHandler(loc)) {
@@ -38,10 +38,16 @@ public abstract class HandlerManager {
 	public static InputStream getFileInputStream(URI loc) throws Exception {
 		for (IRequestHandler handler : handlers) {
 			if (handler.matchesHandler(loc)) {
-				return handler.getFileInputStream(loc);
+				InputStream stream = handler.getFileInputStream(loc);
+				if (stream == null) {
+					throw new Exception("Couldn't find URI: " + loc.toString());
+				} else {
+					return stream;
+				}
 			}
 		}
-		return null;
+		// TODO: specialised exception classes??
+		throw new Exception("No handler available for URI: " + loc.toString());
 	}
 	
 	// To enqueue stuff: