From bf95f03a180188dedeb35b90942b7b744f5a6bbd Mon Sep 17 00:00:00 2001 From: comp500 Date: Sat, 10 Apr 2021 01:45:54 +0100 Subject: [PATCH] Start internal rewrite of file download system --- build.gradle.kts | 4 +- .../infra/packwiz/installer/DownloadTask.kt | 5 +- .../link/infra/packwiz/installer/Main.kt | 3 +- .../infra/packwiz/installer/UpdateManager.kt | 53 +---------- .../installer/metadata/ManifestFile.kt | 4 +- .../packwiz/installer/metadata/ModFile.kt | 4 +- .../packwiz/installer/target/CachedTarget.kt | 23 +++++ .../target/CachedTargetValidation.kt | 95 +++++++++++++++++++ .../infra/packwiz/installer/target/Side.kt | 54 +++++++++++ .../infra/packwiz/installer/target/Target.kt | 32 +++++++ 10 files changed, 217 insertions(+), 60 deletions(-) create mode 100644 src/main/kotlin/link/infra/packwiz/installer/target/CachedTarget.kt create mode 100644 src/main/kotlin/link/infra/packwiz/installer/target/CachedTargetValidation.kt create mode 100644 src/main/kotlin/link/infra/packwiz/installer/target/Side.kt create mode 100644 src/main/kotlin/link/infra/packwiz/installer/target/Target.kt diff --git a/build.gradle.kts b/build.gradle.kts index a24d4b7..08886f9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -126,12 +126,12 @@ if (project.hasProperty("github.token")) { tasks.compileKotlin { kotlinOptions { jvmTarget = "1.8" - freeCompilerArgs = listOf("-Xjvm-default=enable") + freeCompilerArgs = listOf("-Xjvm-default=enable", "-Xallow-result-return-type", "-Xopt-in=kotlin.io.path.ExperimentalPathApi") } } tasks.compileTestKotlin { kotlinOptions { jvmTarget = "1.8" - freeCompilerArgs = listOf("-Xjvm-default=enable") + freeCompilerArgs = listOf("-Xjvm-default=enable", "-Xallow-result-return-type", "-Xopt-in=kotlin.io.path.ExperimentalPathApi") } } \ No newline at end of file diff --git a/src/main/kotlin/link/infra/packwiz/installer/DownloadTask.kt b/src/main/kotlin/link/infra/packwiz/installer/DownloadTask.kt index 120a8c7..d383596 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/DownloadTask.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/DownloadTask.kt @@ -6,6 +6,7 @@ import link.infra.packwiz.installer.metadata.SpaceSafeURI import link.infra.packwiz.installer.metadata.hash.Hash import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash import link.infra.packwiz.installer.metadata.hash.HashUtils.getHasher +import link.infra.packwiz.installer.target.Side import link.infra.packwiz.installer.ui.data.ExceptionDetails import link.infra.packwiz.installer.ui.data.IOptionDetails import link.infra.packwiz.installer.util.Log @@ -18,7 +19,7 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption import java.util.* -internal class DownloadTask private constructor(val metadata: IndexFile.File, defaultFormat: String, private val downloadSide: UpdateManager.Options.Side) : IOptionDetails { +internal class DownloadTask private constructor(val metadata: IndexFile.File, defaultFormat: String, private val downloadSide: Side) : IOptionDetails { var cachedFile: ManifestFile.File? = null private var err: Exception? = null @@ -241,7 +242,7 @@ internal class DownloadTask private constructor(val metadata: IndexFile.File, de companion object { @JvmStatic - fun createTasksFromIndex(index: IndexFile, defaultFormat: String, downloadSide: UpdateManager.Options.Side): List { + fun createTasksFromIndex(index: IndexFile, defaultFormat: String, downloadSide: Side): List { val tasks = ArrayList() for (file in Objects.requireNonNull(index.files)) { tasks.add(DownloadTask(file, defaultFormat, downloadSide)) diff --git a/src/main/kotlin/link/infra/packwiz/installer/Main.kt b/src/main/kotlin/link/infra/packwiz/installer/Main.kt index 77fe877..f22632b 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/Main.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/Main.kt @@ -3,6 +3,7 @@ package link.infra.packwiz.installer import link.infra.packwiz.installer.metadata.SpaceSafeURI +import link.infra.packwiz.installer.target.Side import link.infra.packwiz.installer.ui.cli.CLIHandler import link.infra.packwiz.installer.ui.gui.GUIHandler import link.infra.packwiz.installer.util.Log @@ -68,7 +69,7 @@ class Main(args: Array) { val uOptions = try { UpdateManager.Options.construct( downloadURI = SpaceSafeURI(unparsedArgs[0]), - side = cmd.getOptionValue("side")?.let((UpdateManager.Options.Side)::from), + side = cmd.getOptionValue("side")?.let((Side)::from), packFolder = cmd.getOptionValue("pack-folder"), manifestFile = cmd.getOptionValue("meta-file") ) diff --git a/src/main/kotlin/link/infra/packwiz/installer/UpdateManager.kt b/src/main/kotlin/link/infra/packwiz/installer/UpdateManager.kt index ef7e440..bc92b36 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/UpdateManager.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/UpdateManager.kt @@ -3,7 +3,6 @@ package link.infra.packwiz.installer import com.google.gson.GsonBuilder import com.google.gson.JsonIOException import com.google.gson.JsonSyntaxException -import com.google.gson.annotations.SerializedName import com.moandjiezana.toml.Toml import link.infra.packwiz.installer.DownloadTask.Companion.createTasksFromIndex import link.infra.packwiz.installer.metadata.IndexFile @@ -15,6 +14,7 @@ import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash import link.infra.packwiz.installer.metadata.hash.HashUtils.getHasher import link.infra.packwiz.installer.request.HandlerManager.getFileSource import link.infra.packwiz.installer.request.HandlerManager.getNewLoc +import link.infra.packwiz.installer.target.Side import link.infra.packwiz.installer.ui.IUserInterface import link.infra.packwiz.installer.ui.IUserInterface.CancellationResult import link.infra.packwiz.installer.ui.IUserInterface.ExceptionListResult @@ -55,56 +55,6 @@ class UpdateManager internal constructor(private val opts: Options, val ui: IUse Options(downloadURI, manifestFile ?: "packwiz.json", packFolder ?: ".", side ?: Side.CLIENT) } - enum class Side { - @SerializedName("client") - CLIENT("client"), - @SerializedName("server") - SERVER("server"), - @SerializedName("both") - @Suppress("unused") - BOTH("both", arrayOf(CLIENT, SERVER)); - - private val sideName: String - private val depSides: Array? - - constructor(sideName: String) { - this.sideName = sideName.toLowerCase() - depSides = null - } - - constructor(sideName: String, depSides: Array) { - this.sideName = sideName.toLowerCase() - this.depSides = depSides - } - - override fun toString() = sideName - - fun hasSide(tSide: Side): Boolean { - if (this == tSide) { - return true - } - if (depSides != null) { - for (depSide in depSides) { - if (depSide == tSide) { - return true - } - } - } - return false - } - - companion object { - fun from(name: String): Side? { - val lowerName = name.toLowerCase() - for (side in values()) { - if (side.sideName == lowerName) { - return side - } - } - return null - } - } - } } private fun start() { @@ -444,4 +394,5 @@ class UpdateManager internal constructor(private val opts: Options, val ui: IUse exitProcess(0) } } + } \ No newline at end of file diff --git a/src/main/kotlin/link/infra/packwiz/installer/metadata/ManifestFile.kt b/src/main/kotlin/link/infra/packwiz/installer/metadata/ManifestFile.kt index 180d74f..e87ac36 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/metadata/ManifestFile.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/metadata/ManifestFile.kt @@ -1,15 +1,15 @@ package link.infra.packwiz.installer.metadata import com.google.gson.annotations.JsonAdapter -import link.infra.packwiz.installer.UpdateManager import link.infra.packwiz.installer.metadata.hash.Hash +import link.infra.packwiz.installer.target.Side class ManifestFile { var packFileHash: Hash? = null var indexFileHash: Hash? = null var cachedFiles: MutableMap = HashMap() // If the side changes, EVERYTHING invalidates. FUN!!! - var cachedSide = UpdateManager.Options.Side.CLIENT + var cachedSide = Side.CLIENT // TODO: switch to Kotlin-friendly JSON/TOML libs? class File { diff --git a/src/main/kotlin/link/infra/packwiz/installer/metadata/ModFile.kt b/src/main/kotlin/link/infra/packwiz/installer/metadata/ModFile.kt index b84271d..7767e72 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/metadata/ModFile.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/metadata/ModFile.kt @@ -1,17 +1,17 @@ package link.infra.packwiz.installer.metadata import com.google.gson.annotations.SerializedName -import link.infra.packwiz.installer.UpdateManager import link.infra.packwiz.installer.metadata.hash.Hash import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash import link.infra.packwiz.installer.request.HandlerManager.getFileSource import link.infra.packwiz.installer.request.HandlerManager.getNewLoc +import link.infra.packwiz.installer.target.Side import okio.Source class ModFile { var name: String? = null var filename: String? = null - var side: UpdateManager.Options.Side? = null + var side: Side? = null var download: Download? = null class Download { diff --git a/src/main/kotlin/link/infra/packwiz/installer/target/CachedTarget.kt b/src/main/kotlin/link/infra/packwiz/installer/target/CachedTarget.kt new file mode 100644 index 0000000..edf9112 --- /dev/null +++ b/src/main/kotlin/link/infra/packwiz/installer/target/CachedTarget.kt @@ -0,0 +1,23 @@ +package link.infra.packwiz.installer.target + +import link.infra.packwiz.installer.metadata.hash.Hash +import java.nio.file.Path + +data class CachedTarget( + /** + * @see Target.name + */ + val name: String, + /** + * The location where the target was last downloaded to. + * This is used for removing old files when the destination path changes. + * This shouldn't be set to the .disabled path (that is manually appended and checked) + */ + val cachedLocation: Path, + val enabled: Boolean, + val hash: Hash, + /** + * For detecting when a target transitions non-optional -> optional and showing the option selection screen + */ + val isOptional: Boolean +) diff --git a/src/main/kotlin/link/infra/packwiz/installer/target/CachedTargetValidation.kt b/src/main/kotlin/link/infra/packwiz/installer/target/CachedTargetValidation.kt new file mode 100644 index 0000000..fec7528 --- /dev/null +++ b/src/main/kotlin/link/infra/packwiz/installer/target/CachedTargetValidation.kt @@ -0,0 +1,95 @@ +package link.infra.packwiz.installer.target + +import java.io.IOException +import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributes +import kotlin.io.path.relativeTo + +data class CachedTargetStatus(val target: CachedTarget, var isValid: Boolean, var markDisabled: Boolean) + +fun validate(targets: List, baseDir: Path) = runCatching { + val results = targets.map { + CachedTargetStatus(it, isValid = false, markDisabled = false) + } + val tree = buildTree(results, baseDir) + + // Efficient file exists checking using directory listing, several orders of magnitude faster than Files.exists calls + Files.walkFileTree(baseDir, setOf(FileVisitOption.FOLLOW_LINKS), Int.MAX_VALUE, object : FileVisitor { + var currentNode: PathNode = tree + + override fun preVisitDirectory(dir: Path?, attrs: BasicFileAttributes?): FileVisitResult { + if (dir == null) { + return FileVisitResult.SKIP_SUBTREE + } + val subdirNode = currentNode.subdirs[dir.getName(dir.nameCount - 1)] + return if (subdirNode != null) { + currentNode = subdirNode + FileVisitResult.CONTINUE + } else { + FileVisitResult.SKIP_SUBTREE + } + } + + override fun visitFile(file: Path?, attrs: BasicFileAttributes?): FileVisitResult { + if (file == null) { + return FileVisitResult.CONTINUE + } + // TODO: these are relative paths to baseDir + // TODO: strip the .disabled for lookup + val target = currentNode.files[file.getName(file.nameCount - 1)] + if (target != null) { + val disabledFile = file.endsWith(".disabled") + // If a .disabled file and the actual file both exist, mark as invalid if the target is disabled + if ((disabledFile )) { + + } + } + return FileVisitResult.CONTINUE + } + + @Throws(IOException::class) + override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult { + if (exc != null) { + throw exc + } + throw IOException("visitFileFailed called with no exception") + } + + @Throws(IOException::class) + override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult { + if (exc != null) { + throw exc + } else { + val parent = currentNode.parent + if (parent != null) { + currentNode = parent + } else { + throw IOException("Invalid visitor tree structure") + } + return FileVisitResult.CONTINUE + } + } + }) + + results +} + +fun buildTree(targets: List, baseDir: Path): PathNode { + val root = PathNode() + for (target in targets) { + val relPath = target.target.cachedLocation.relativeTo(baseDir) + var node = root + // Traverse all the directory components, except for the last one + for (i in 0 until (relPath.nameCount - 1)) { + node = node.createSubdir(relPath.getName(i)) + } + node.files[relPath.getName(relPath.nameCount - 1)] = target + } + return root +} + +data class PathNode(val subdirs: MutableMap>, val files: MutableMap, val parent: PathNode?) { + constructor() : this(mutableMapOf(), mutableMapOf(), null) + + fun createSubdir(nextComponent: Path) = subdirs.getOrPut(nextComponent, { PathNode(mutableMapOf(), mutableMapOf(), this) }) +} \ No newline at end of file diff --git a/src/main/kotlin/link/infra/packwiz/installer/target/Side.kt b/src/main/kotlin/link/infra/packwiz/installer/target/Side.kt new file mode 100644 index 0000000..e0a4d34 --- /dev/null +++ b/src/main/kotlin/link/infra/packwiz/installer/target/Side.kt @@ -0,0 +1,54 @@ +package link.infra.packwiz.installer.target + +import com.google.gson.annotations.SerializedName + +enum class Side { + @SerializedName("client") + CLIENT("client"), + @SerializedName("server") + SERVER("server"), + @SerializedName("both") + @Suppress("unused") + BOTH("both", arrayOf(CLIENT, SERVER)); + + private val sideName: String + private val depSides: Array? + + constructor(sideName: String) { + this.sideName = sideName.toLowerCase() + depSides = null + } + + constructor(sideName: String, depSides: Array) { + this.sideName = sideName.toLowerCase() + this.depSides = depSides + } + + override fun toString() = sideName + + fun hasSide(tSide: Side): Boolean { + if (this == tSide) { + return true + } + if (depSides != null) { + for (depSide in depSides) { + if (depSide == tSide) { + return true + } + } + } + return false + } + + companion object { + fun from(name: String): Side? { + val lowerName = name.toLowerCase() + for (side in values()) { + if (side.sideName == lowerName) { + return side + } + } + return null + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/link/infra/packwiz/installer/target/Target.kt b/src/main/kotlin/link/infra/packwiz/installer/target/Target.kt new file mode 100644 index 0000000..f8a623a --- /dev/null +++ b/src/main/kotlin/link/infra/packwiz/installer/target/Target.kt @@ -0,0 +1,32 @@ +package link.infra.packwiz.installer.target + +import link.infra.packwiz.installer.metadata.SpaceSafeURI +import link.infra.packwiz.installer.metadata.hash.Hash +import java.nio.file.Path + +data class Target( + /** + * The name that uniquely identifies this target. + * Often equal to the name of the metadata file for this target, and can be displayed to the user in progress UI. + */ + val name: String, + /** + * An optional user-friendly name. + */ + val userFriendlyName: String?, + val src: SpaceSafeURI, + val dest: Path, + val hash: Hash, + val side: Side, + val optional: Boolean, + val optionalDefaultValue: Boolean, + val optionalDescription: String, + /** + * If this is true, don't update a target when the file already exists. + */ + val noOverwrite: Boolean +) { + fun Iterable.filterForSide(side: Side) = this.filter { + it.side.hasSide(side) + } +}