From 81c1ebaa15ee8158e4daa810e0912131119f65e8 Mon Sep 17 00:00:00 2001 From: comp500 Date: Fri, 21 Jun 2019 14:28:41 +0100 Subject: [PATCH] Commit WIP downloading process --- .../packwiz/installer/UpdateManager.java | 173 +++++++++++++++++- .../packwiz/installer/metadata/IndexFile.java | 85 ++++++++- .../installer/metadata/ManifestFile.java | 7 + .../packwiz/installer/metadata/ModFile.java | 40 +++- .../packwiz/installer/metadata/PackFile.java | 3 +- .../packwiz/installer/metadata/hash/Hash.java | 8 +- .../installer/request/HandlerManager.java | 12 +- 7 files changed, 313 insertions(+), 15 deletions(-) diff --git a/src/main/java/link/infra/packwiz/installer/UpdateManager.java b/src/main/java/link/infra/packwiz/installer/UpdateManager.java index 05c9446..37568c8 100644 --- a/src/main/java/link/infra/packwiz/installer/UpdateManager.java +++ b/src/main/java/link/infra/packwiz/installer/UpdateManager.java @@ -7,7 +7,18 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; import com.google.gson.Gson; import com.google.gson.JsonIOException; @@ -15,6 +26,7 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.annotations.SerializedName; import com.moandjiezana.toml.Toml; +import link.infra.packwiz.installer.metadata.IndexFile; import link.infra.packwiz.installer.metadata.ManifestFile; import link.infra.packwiz.installer.metadata.PackFile; import link.infra.packwiz.installer.metadata.hash.Hash; @@ -35,10 +47,8 @@ public class UpdateManager { public static enum Side { @SerializedName("client") - CLIENT("client"), - @SerializedName("server") - SERVER("server"), - @SerializedName("both") + CLIENT("client"), @SerializedName("server") + SERVER("server"), @SerializedName("both") BOTH("both", new Side[] { CLIENT, SERVER }); private final String sideName; @@ -138,6 +148,8 @@ public class UpdateManager { System.out.println(pf.name); + processIndex(HandlerManager.getNewLoc(opts.downloadURI, pf.index.file), + new Hash(pf.index.hash, pf.index.hashFormat), manifest); // When successfully updated manifest.packFileHash = packFileHash; @@ -149,10 +161,161 @@ public class UpdateManager { // TODO: add message? ui.handleException(e); } - + } protected void checkOptions() { // TODO: implement } + + protected void processIndex(URI indexUri, Hash indexHash, ManifestFile manifest) { + Hash.HashInputStream indexFileStream; + try { + InputStream stream = HandlerManager.getFileInputStream(opts.downloadURI); + if (stream == null) { + throw new Exception("Index file URI is invalid, is it supported?"); + } + indexFileStream = new Hash.HashInputStream(stream, indexHash); + } catch (Exception e) { + // TODO: still launch the game if updating doesn't work? + // TODO: ask user if they want to launch the game, exit(1) if they don't + ui.handleExceptionAndExit(e); + return; + } + IndexFile indexFile; + try { + indexFile = new Toml().read(indexFileStream).to(IndexFile.class); + } catch (IllegalStateException e) { + ui.handleExceptionAndExit(e); + return; + } + + if (!indexFileStream.hashIsEqual()) { + System.out.println("Hash problems!!!!!!!"); + // TODO: throw exception + } + + // TODO: progress bar + ConcurrentLinkedQueue exceptionQueue = new ConcurrentLinkedQueue(); + List newFiles = indexFile.files.stream().filter(f -> { + ManifestFile.File cachedFile = manifest.cachedFiles.get(f.file); + Hash newHash = new Hash(f.hash, f.hashFormat); + return cachedFile == null || newHash.equals(cachedFile.hash); + }).parallel().map(f -> { + try { + f.downloadMeta(indexFile, indexUri); + } catch (Exception e) { + exceptionQueue.add(e); + } + return f; + }).collect(Collectors.toList()); + + for (Exception e : exceptionQueue) { + // TODO: collect all exceptions, present in one dialog + ui.handleException(e); + } + + // TODO: present options + // TODO: all options should be presented, not just new files!!!!!!! + // and options should be readded to newFiles after option -> true + newFiles.stream().filter(f -> f.linkedFile != null).filter(f -> f.linkedFile.option != null).map(f -> { + return "option: " + (f.linkedFile.option.description == null ? "null" : f.linkedFile.option.description); + }).forEachOrdered(desc -> { + System.out.println(desc); + }); + + // TODO: different thread pool type? + ExecutorService threadPool = Executors.newFixedThreadPool(10); + CompletionService completionService = new ExecutorCompletionService( + threadPool); + + for (IndexFile.File f : newFiles) { + ManifestFile.File cachedFile = manifest.cachedFiles.get(f.file); + completionService.submit(new Callable() { + public DownloadCompletion call() { + DownloadCompletion dc = new DownloadCompletion(); + dc.file = f; + + if (cachedFile.linkedFileHash != null && f.linkedFile != null) { + try { + if (cachedFile.linkedFileHash.equals(f.linkedFile.getHash())) { + // Do nothing, the file didn't change + // TODO: but if the hash of the metafile changed, what did change????? + // should this be checked somehow?? + return dc; + } + } catch (Exception e) {} + } + + try { + InputStream stream = f.getInputStream(indexUri); + if (stream == null) { + throw new Exception("Failed to open download stream"); + } + Hash.HashInputStream fileStream = new Hash.HashInputStream(stream, f.getHash()); + // UGHHHHHH if you're reading this point in history + // this is the point where i change EVERYTHING to okio because it's 1000000000000 times nicer for this + byte[] data = fileStream.readAllBytes(); + Files.copy(fileStream, Paths.get(opts.packFolder, f.getDestURI().toString()), StandardCopyOption.REPLACE_EXISTING); + + return dc; + } catch (Exception e) { + dc.err = e; + return dc; + } + } + }); + } + + for (int i = 0; i < newFiles.size(); i++) { + DownloadCompletion ret; + try { + ret = completionService.take().get(); + } catch (InterruptedException | ExecutionException e) { + // TODO: collect all exceptions, present in one dialog + ui.handleException(e); + ret = null; + } + // Update manifest + if (ret != null && ret.err == null && ret.file != null) { + ManifestFile.File newCachedFile = new ManifestFile.File(); + try { + newCachedFile.hash = ret.file.getHash(); + if (newCachedFile.hash == null) { + throw new Exception("Invalid hash!"); + } + } catch (Exception e) { + ret.err = e; + } + if (ret.file.metafile && ret.file.linkedFile != null) { + newCachedFile.isOptional = ret.file.linkedFile.isOptional(); + if (newCachedFile.isOptional) { + newCachedFile.optionValue = ret.file.optionValue; + } + try { + newCachedFile.linkedFileHash = ret.file.linkedFile.getHash(); + } catch (Exception e) { + ret.err = e; + } + } + } + // TODO: show errors properly? + String progress; + if (ret != null && ret.file != null) { + progress = "Downloaded " + ret.file.getName(); + } else if (ret.err != null) { + progress = "Failed to download: " + ret.err.getMessage(); + } else { + progress = "Failed to download, unknown reason"; + } + ui.submitProgress(new InstallProgress(progress, i + 1, newFiles.size())); + } + // option = false file hashes should be stored to disk, but not downloaded + // TODO: don't include optional files in progress???? + } + + private class DownloadCompletion { + Exception err; + IndexFile.File file; + } } diff --git a/src/main/java/link/infra/packwiz/installer/metadata/IndexFile.java b/src/main/java/link/infra/packwiz/installer/metadata/IndexFile.java index b14cdb6..29de8c7 100644 --- a/src/main/java/link/infra/packwiz/installer/metadata/IndexFile.java +++ b/src/main/java/link/infra/packwiz/installer/metadata/IndexFile.java @@ -1,8 +1,15 @@ package link.infra.packwiz.installer.metadata; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Paths; import java.util.List; import com.google.gson.annotations.SerializedName; +import com.moandjiezana.toml.Toml; + +import link.infra.packwiz.installer.metadata.hash.Hash; +import link.infra.packwiz.installer.request.HandlerManager; public class IndexFile { @SerializedName("hash-format") @@ -10,12 +17,88 @@ public class IndexFile { public List files; public static class File { - public String file; + public URI file; @SerializedName("hash-format") public String hashFormat; public String hash; + // TODO: implement public String alias; public boolean metafile; + // TODO: implement public boolean preserve; + + public transient ModFile linkedFile; + public transient URI linkedFileURI; + public transient boolean optionValue = true; + + public void downloadMeta(IndexFile parentIndexFile, URI indexUri) throws Exception { + if (!metafile) { + return; + } + if (hashFormat == null || hashFormat.length() == 0) { + hashFormat = parentIndexFile.hashFormat; + } + Hash fileHash = new Hash(hash, hashFormat); + linkedFileURI = HandlerManager.getNewLoc(indexUri, file); + InputStream stream = HandlerManager.getFileInputStream(linkedFileURI); + if (stream == null) { + throw new Exception("Index file URI is invalid, is it supported?"); + } + Hash.HashInputStream fileStream = new Hash.HashInputStream(stream, fileHash); + + linkedFile = new Toml().read(fileStream).to(ModFile.class); + if (!fileStream.hashIsEqual()) { + throw new Exception("Invalid mod file hash"); + } + } + + public InputStream getInputStream(URI indexUri) throws Exception { + if (metafile) { + if (linkedFile == null) { + throw new Exception("Linked file doesn't exist!"); + } + return linkedFile.getInputStream(linkedFileURI); + } else { + URI newLoc = HandlerManager.getNewLoc(indexUri, file); + if (newLoc == null) { + throw new Exception("Index file URI is invalid"); + } + return HandlerManager.getFileInputStream(newLoc); + } + } + + public Hash getHash() throws Exception { + if (hash == null) { + throw new Exception("Index file doesn't have a hash"); + } + if (hashFormat == null) { + throw new Exception("Index file doesn't have a hash format"); + } + return new Hash(hash, hashFormat); + } + + public String getName() { + if (metafile) { + if (linkedFile != null) { + if (linkedFile.name != null) { + return linkedFile.name; + } else if (linkedFile.filename != null) { + return linkedFile.filename; + } + } + } + if (file != null) { + return Paths.get(file.getPath()).getFileName().toString(); + } + return file.getPath(); + } + + public URI getDestURI() { + if (metafile && linkedFile != null) { + return file.resolve(linkedFile.filename); + } else { + return file; + } + } } } \ No newline at end of file diff --git a/src/main/java/link/infra/packwiz/installer/metadata/ManifestFile.java b/src/main/java/link/infra/packwiz/installer/metadata/ManifestFile.java index 826f44b..ecf5c65 100644 --- a/src/main/java/link/infra/packwiz/installer/metadata/ManifestFile.java +++ b/src/main/java/link/infra/packwiz/installer/metadata/ManifestFile.java @@ -1,13 +1,20 @@ package link.infra.packwiz.installer.metadata; +import java.net.URI; +import java.util.Map; + import link.infra.packwiz.installer.metadata.hash.Hash; public class ManifestFile { public Hash packFileHash = null; public Hash indexFileHash = null; + public Map cachedFiles; public static class File { public Hash hash = null; + public boolean isOptional = false; + public boolean optionValue = true; + public Hash linkedFileHash = null; } } \ No newline at end of file diff --git a/src/main/java/link/infra/packwiz/installer/metadata/ModFile.java b/src/main/java/link/infra/packwiz/installer/metadata/ModFile.java index 1795b20..1f536da 100644 --- a/src/main/java/link/infra/packwiz/installer/metadata/ModFile.java +++ b/src/main/java/link/infra/packwiz/installer/metadata/ModFile.java @@ -1,13 +1,16 @@ package link.infra.packwiz.installer.metadata; +import java.io.InputStream; import java.net.URI; import java.util.Map; import com.google.gson.annotations.SerializedName; import link.infra.packwiz.installer.UpdateManager.Options.Side; +import link.infra.packwiz.installer.metadata.hash.Hash; +import link.infra.packwiz.installer.request.HandlerManager; -class ModFile { +public class ModFile { public String name; public String filename; public Side side; @@ -30,4 +33,39 @@ class ModFile { public boolean defaultValue; } + public InputStream getInputStream(URI baseLoc) throws Exception { + if (download == null) { + throw new Exception("Metadata file doesn't have download"); + } + if (download.url == null) { + throw new Exception("Metadata file doesn't have a download URI"); + } + URI newLoc = HandlerManager.getNewLoc(baseLoc, download.url); + if (newLoc == null) { + throw new Exception("Metadata file URI is invalid"); + } + + return HandlerManager.getFileInputStream(newLoc); + } + + public Hash getHash() throws Exception { + if (download == null) { + throw new Exception("Metadata file doesn't have download"); + } + if (download.hash == null) { + throw new Exception("Metadata file doesn't have a hash"); + } + if (download.hashFormat == null) { + throw new Exception("Metadata file doesn't have a hash format"); + } + return new Hash(download.hash, download.hashFormat); + } + + public boolean isOptional() { + if (option != null) { + return option.optional; + } + return false; + } + } \ No newline at end of file diff --git a/src/main/java/link/infra/packwiz/installer/metadata/PackFile.java b/src/main/java/link/infra/packwiz/installer/metadata/PackFile.java index d112b41..0924cd2 100644 --- a/src/main/java/link/infra/packwiz/installer/metadata/PackFile.java +++ b/src/main/java/link/infra/packwiz/installer/metadata/PackFile.java @@ -1,5 +1,6 @@ package link.infra.packwiz.installer.metadata; +import java.net.URI; import java.util.Map; import com.google.gson.annotations.SerializedName; @@ -9,7 +10,7 @@ public class PackFile { public IndexFileLoc index; public static class IndexFileLoc { - public String file; + public URI file; @SerializedName("hash-format") public String hashFormat; public String hash; diff --git a/src/main/java/link/infra/packwiz/installer/metadata/hash/Hash.java b/src/main/java/link/infra/packwiz/installer/metadata/hash/Hash.java index 510602a..fd2a009 100644 --- a/src/main/java/link/infra/packwiz/installer/metadata/hash/Hash.java +++ b/src/main/java/link/infra/packwiz/installer/metadata/hash/Hash.java @@ -104,11 +104,11 @@ public class Hash { return output; } - public boolean isNew() { - if (output != null) { - return !output.equals(compare); + public boolean hashIsEqual() { + if (output == null) { + get(); } - return false; + return !output.equals(compare); } } diff --git a/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java b/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java index 31f8ab8..a44ea62 100644 --- a/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java +++ b/src/main/java/link/infra/packwiz/installer/request/HandlerManager.java @@ -18,10 +18,10 @@ public abstract class HandlerManager { } public static URI getNewLoc(URI base, URI loc) { + if (loc == null) return null; if (base != null) { loc = base.resolve(loc); } - if (loc == null) return null; for (IRequestHandler handler : handlers) { if (handler.matchesHandler(loc)) { @@ -38,10 +38,16 @@ public abstract class HandlerManager { public static InputStream getFileInputStream(URI loc) throws Exception { for (IRequestHandler handler : handlers) { if (handler.matchesHandler(loc)) { - return handler.getFileInputStream(loc); + InputStream stream = handler.getFileInputStream(loc); + if (stream == null) { + throw new Exception("Couldn't find URI: " + loc.toString()); + } else { + return stream; + } } } - return null; + // TODO: specialised exception classes?? + throw new Exception("No handler available for URI: " + loc.toString()); } // To enqueue stuff: