From fd87edd6cad31e725fa6115c21edd91bec7aabbf Mon Sep 17 00:00:00 2001 From: comp500 Date: Sat, 1 Jun 2019 16:26:15 +0100 Subject: [PATCH] Make github request handling work, fix some things --- .../handlers/RequestHandlerGithub.java | 70 ++++++++++++++++-- .../request/handlers/RequestHandlerHTTP.java | 4 +- .../request/handlers/RequestHandlerZip.java | 71 ++++++++++++++++++- 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java index 009b828..32997ba 100644 --- a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerGithub.java @@ -1,25 +1,83 @@ package link.infra.packwiz.installer.request.handlers; import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class RequestHandlerGithub extends RequestHandlerZip { + + public RequestHandlerGithub() { + super(true); + } + + @Override + public URI getNewLoc(URI loc) { + return loc; + } + + // TODO: is caching really needed, if HTTPURLConnection follows redirects correctly? + private Map zipUriMap = new HashMap(); + final ReentrantReadWriteLock zipUriLock = new ReentrantReadWriteLock(); + private static Pattern repoMatcherPattern = Pattern.compile("/([\\w.-]+/[\\w.-]+).*"); + + private String getRepoName(URI loc) { + Matcher matcher = repoMatcherPattern.matcher(loc.getPath()); + matcher.matches(); + return matcher.group(1); + } @Override protected URI getZipUri(URI loc) throws Exception { - // TODO Auto-generated method stub - return null; + String repoName = getRepoName(loc); + String branchName = getBranch(loc); + zipUriLock.readLock().lock(); + URI zipUri = zipUriMap.get(repoName + "/" + branchName); + zipUriLock.readLock().unlock(); + if (zipUri != null) { + return zipUri; + } + + zipUri = new URI("https://api.github.com/repos/" + repoName + "/zipball/" + branchName); + + zipUriLock.writeLock().lock(); + // If another thread sets the value concurrently, use the value of the + // thread that first acquired the lock. + URI zipUriInserted = zipUriMap.putIfAbsent(repoName + "/" + branchName, zipUri); + if (zipUriInserted != null) { + zipUri = zipUriInserted; + } + zipUriLock.writeLock().unlock(); + return zipUri; + } + + private static Pattern branchMatcherPattern = Pattern.compile("/[\\w.-]+/[\\w.-]+/blob/([\\w.-]+).*"); + + private String getBranch(URI loc) { + Matcher matcher = branchMatcherPattern.matcher(loc.getPath()); + matcher.matches(); + return matcher.group(1); } @Override protected URI getLocationInZip(URI loc) throws Exception { - // TODO Auto-generated method stub - return null; + String path = "/" + getRepoName(loc) + "/blob/" + getBranch(loc); + return new URI(loc.getScheme(), loc.getAuthority(), path, null, null).relativize(loc); } @Override public boolean matchesHandler(URI loc) { - // TODO Auto-generated method stub - return false; + String scheme = loc.getScheme(); + if (!(scheme.equals("http") || scheme.equals("https"))) { + return false; + } + if (!loc.getHost().equals("github.com")) { + return false; + } + // TODO: sanity checks, support for more github urls + return true; } } diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java index 1cde9f9..db6d3da 100644 --- a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerHTTP.java @@ -17,7 +17,9 @@ public class RequestHandlerHTTP implements IRequestHandler { @Override public InputStream getFileInputStream(URI loc) throws Exception { URLConnection conn = loc.toURL().openConnection(); - conn.addRequestProperty("Accept", "application/octet-stream"); + // TODO: when do we send specific headers??? should there be a way to signal this? + // github *sometimes* requires it, sometimes not! + //conn.addRequestProperty("Accept", "application/octet-stream"); // 30 second read timeout conn.setReadTimeout(30 * 1000); return conn.getInputStream(); diff --git a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java index f468ee8..b32276b 100644 --- a/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java +++ b/src/main/java/link/infra/packwiz/installer/request/handlers/RequestHandlerZip.java @@ -10,11 +10,26 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Predicate; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; public abstract class RequestHandlerZip extends RequestHandlerHTTP { + protected final boolean modeHasFolder; + + public RequestHandlerZip(boolean modeHasFolder) { + this.modeHasFolder = modeHasFolder; + } + + private String removeFolder(String name) { + if (modeHasFolder) { + return name.substring(name.indexOf("/")+1); + } else { + return name; + } + } + private class ZipReader { private final ZipInputStream zis; @@ -43,10 +58,11 @@ public abstract class RequestHandlerZip extends RequestHandlerHTTP { return null; } byte[] data = readCurrFile(); - if (loc.equals(new URI(entry.getName()))) { + URI fileLoc = new URI(removeFolder(entry.getName())); + if (loc.equals(fileLoc)) { return data; } else { - readFiles.put(loc, data); + readFiles.put(fileLoc, data); } } } @@ -68,6 +84,31 @@ public abstract class RequestHandlerZip extends RequestHandlerHTTP { return null; } + public URI findInZip(Predicate matches) throws Exception { + filesLock.lock(); + for (URI file : readFiles.keySet()) { + if (matches.test(file)) { + filesLock.unlock(); + return file; + } + } + + while (true) { + entry = zis.getNextEntry(); + if (entry == null) { + filesLock.unlock(); + return null; + } + byte[] data = readCurrFile(); + URI fileLoc = new URI(removeFolder(entry.getName())); + readFiles.put(fileLoc, data); + if (matches.test(fileLoc)) { + filesLock.unlock(); + return fileLoc; + } + } + } + } private final Map cache = new HashMap(); @@ -91,7 +132,12 @@ public abstract class RequestHandlerZip extends RequestHandlerHTTP { // Recheck, because unlocking read lock allows another thread to modify it zr = cache.get(zipUri); if (zr == null) { - zr = new ZipReader(super.getFileInputStream(zipUri)); + InputStream str = super.getFileInputStream(zipUri); + if (str == null) { + cacheLock.writeLock().unlock(); + return null; + } + zr = new ZipReader(str); cache.put(zipUri, zr); } cacheLock.writeLock().unlock(); @@ -99,5 +145,24 @@ public abstract class RequestHandlerZip extends RequestHandlerHTTP { return zr.getFileInputStream(getLocationInZip(loc)); } + + protected URI findInZip(URI loc, Predicate matches) throws Exception { + URI zipUri = getZipUri(loc); + cacheLock.readLock().lock(); + ZipReader zr = cache.get(zipUri); + cacheLock.readLock().unlock(); + if (zr == null) { + cacheLock.writeLock().lock(); + // Recheck, because unlocking read lock allows another thread to modify it + zr = cache.get(zipUri); + if (zr == null) { + zr = new ZipReader(super.getFileInputStream(zipUri)); + cache.put(zipUri, zr); + } + cacheLock.writeLock().unlock(); + } + + return zr.findInZip(matches); + } }