Start internal rewrite of file download system

build.gradle.kts

@@ -126,12 +126,12 @@ if (project.hasProperty("github.token")) {
 tasks.compileKotlin {
 	kotlinOptions {
 		jvmTarget = "1.8"
-		freeCompilerArgs = listOf("-Xjvm-default=enable")
+		freeCompilerArgs = listOf("-Xjvm-default=enable", "-Xallow-result-return-type", "-Xopt-in=kotlin.io.path.ExperimentalPathApi")
 	}
 }
 tasks.compileTestKotlin {
 	kotlinOptions {
 		jvmTarget = "1.8"
-		freeCompilerArgs = listOf("-Xjvm-default=enable")
+		freeCompilerArgs = listOf("-Xjvm-default=enable", "-Xallow-result-return-type", "-Xopt-in=kotlin.io.path.ExperimentalPathApi")
 	}
 }

src/main/kotlin/link/infra/packwiz/installer/DownloadTask.kt

@@ -6,6 +6,7 @@ import link.infra.packwiz.installer.metadata.SpaceSafeURI
 import link.infra.packwiz.installer.metadata.hash.Hash
 import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash
 import link.infra.packwiz.installer.metadata.hash.HashUtils.getHasher
+import link.infra.packwiz.installer.target.Side
 import link.infra.packwiz.installer.ui.data.ExceptionDetails
 import link.infra.packwiz.installer.ui.data.IOptionDetails
 import link.infra.packwiz.installer.util.Log
@@ -18,7 +19,7 @@ import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 import java.util.*
 
-internal class DownloadTask private constructor(val metadata: IndexFile.File, defaultFormat: String, private val downloadSide: UpdateManager.Options.Side) : IOptionDetails {
+internal class DownloadTask private constructor(val metadata: IndexFile.File, defaultFormat: String, private val downloadSide: Side) : IOptionDetails {
 	var cachedFile: ManifestFile.File? = null
 
 	private var err: Exception? = null
@@ -241,7 +242,7 @@ internal class DownloadTask private constructor(val metadata: IndexFile.File, de
 
 	companion object {
 		@JvmStatic
-		fun createTasksFromIndex(index: IndexFile, defaultFormat: String, downloadSide: UpdateManager.Options.Side): List<DownloadTask> {
+		fun createTasksFromIndex(index: IndexFile, defaultFormat: String, downloadSide: Side): List<DownloadTask> {
 			val tasks = ArrayList<DownloadTask>()
 			for (file in Objects.requireNonNull(index.files)) {
 				tasks.add(DownloadTask(file, defaultFormat, downloadSide))

src/main/kotlin/link/infra/packwiz/installer/Main.kt

@@ -3,6 +3,7 @@
 package link.infra.packwiz.installer
 
 import link.infra.packwiz.installer.metadata.SpaceSafeURI
+import link.infra.packwiz.installer.target.Side
 import link.infra.packwiz.installer.ui.cli.CLIHandler
 import link.infra.packwiz.installer.ui.gui.GUIHandler
 import link.infra.packwiz.installer.util.Log
@@ -68,7 +69,7 @@ class Main(args: Array<String>) {
 		val uOptions = try {
 			UpdateManager.Options.construct(
 				downloadURI = SpaceSafeURI(unparsedArgs[0]),
-				side = cmd.getOptionValue("side")?.let((UpdateManager.Options.Side)::from),
+				side = cmd.getOptionValue("side")?.let((Side)::from),
 				packFolder = cmd.getOptionValue("pack-folder"),
 				manifestFile = cmd.getOptionValue("meta-file")
 			)

src/main/kotlin/link/infra/packwiz/installer/UpdateManager.kt

@@ -3,7 +3,6 @@ package link.infra.packwiz.installer
 import com.google.gson.GsonBuilder
 import com.google.gson.JsonIOException
 import com.google.gson.JsonSyntaxException
-import com.google.gson.annotations.SerializedName
 import com.moandjiezana.toml.Toml
 import link.infra.packwiz.installer.DownloadTask.Companion.createTasksFromIndex
 import link.infra.packwiz.installer.metadata.IndexFile
@@ -15,6 +14,7 @@ import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash
 import link.infra.packwiz.installer.metadata.hash.HashUtils.getHasher
 import link.infra.packwiz.installer.request.HandlerManager.getFileSource
 import link.infra.packwiz.installer.request.HandlerManager.getNewLoc
+import link.infra.packwiz.installer.target.Side
 import link.infra.packwiz.installer.ui.IUserInterface
 import link.infra.packwiz.installer.ui.IUserInterface.CancellationResult
 import link.infra.packwiz.installer.ui.IUserInterface.ExceptionListResult
@@ -55,56 +55,6 @@ class UpdateManager internal constructor(private val opts: Options, val ui: IUse
 				Options(downloadURI, manifestFile ?: "packwiz.json", packFolder ?: ".", side ?: Side.CLIENT)
 		}
 
-		enum class Side {
-			@SerializedName("client")
-			CLIENT("client"),
-			@SerializedName("server")
-			SERVER("server"),
-			@SerializedName("both")
-			@Suppress("unused")
-			BOTH("both", arrayOf(CLIENT, SERVER));
-
-			private val sideName: String
-			private val depSides: Array<Side>?
-
-			constructor(sideName: String) {
-				this.sideName = sideName.toLowerCase()
-				depSides = null
-			}
-
-			constructor(sideName: String, depSides: Array<Side>) {
-				this.sideName = sideName.toLowerCase()
-				this.depSides = depSides
-			}
-
-			override fun toString() = sideName
-
-			fun hasSide(tSide: Side): Boolean {
-				if (this == tSide) {
-					return true
-				}
-				if (depSides != null) {
-					for (depSide in depSides) {
-						if (depSide == tSide) {
-							return true
-						}
-					}
-				}
-				return false
-			}
-
-			companion object {
-				fun from(name: String): Side? {
-					val lowerName = name.toLowerCase()
-					for (side in values()) {
-						if (side.sideName == lowerName) {
-							return side
-						}
-					}
-					return null
-				}
-			}
-		}
 	}
 
 	private fun start() {
@@ -444,4 +394,5 @@ class UpdateManager internal constructor(private val opts: Options, val ui: IUse
 			exitProcess(0)
 		}
 	}
+
 }

src/main/kotlin/link/infra/packwiz/installer/metadata/ManifestFile.kt

@@ -1,15 +1,15 @@
 package link.infra.packwiz.installer.metadata
 
 import com.google.gson.annotations.JsonAdapter
-import link.infra.packwiz.installer.UpdateManager
 import link.infra.packwiz.installer.metadata.hash.Hash
+import link.infra.packwiz.installer.target.Side
 
 class ManifestFile {
 	var packFileHash: Hash? = null
 	var indexFileHash: Hash? = null
 	var cachedFiles: MutableMap<SpaceSafeURI, File> = HashMap()
 	// If the side changes, EVERYTHING invalidates. FUN!!!
-	var cachedSide = UpdateManager.Options.Side.CLIENT
+	var cachedSide = Side.CLIENT
 
 	// TODO: switch to Kotlin-friendly JSON/TOML libs?
 	class File {

src/main/kotlin/link/infra/packwiz/installer/metadata/ModFile.kt

@@ -1,17 +1,17 @@
 package link.infra.packwiz.installer.metadata
 
 import com.google.gson.annotations.SerializedName
-import link.infra.packwiz.installer.UpdateManager
 import link.infra.packwiz.installer.metadata.hash.Hash
 import link.infra.packwiz.installer.metadata.hash.HashUtils.getHash
 import link.infra.packwiz.installer.request.HandlerManager.getFileSource
 import link.infra.packwiz.installer.request.HandlerManager.getNewLoc
+import link.infra.packwiz.installer.target.Side
 import okio.Source
 
 class ModFile {
 	var name: String? = null
 	var filename: String? = null
-	var side: UpdateManager.Options.Side? = null
+	var side: Side? = null
 	var download: Download? = null
 
 	class Download {

src/main/kotlin/link/infra/packwiz/installer/target/CachedTarget.kt

@@ -0,0 +1,23 @@
+package link.infra.packwiz.installer.target
+
+import link.infra.packwiz.installer.metadata.hash.Hash
+import java.nio.file.Path
+
+data class CachedTarget(
+	/**
+	 * @see Target.name
+	 */
+	val name: String,
+	/**
+	 * The location where the target was last downloaded to.
+	 * This is used for removing old files when the destination path changes.
+	 * This shouldn't be set to the .disabled path (that is manually appended and checked)
+	 */
+	val cachedLocation: Path,
+	val enabled: Boolean,
+	val hash: Hash,
+	/**
+	 * For detecting when a target transitions non-optional -> optional and showing the option selection screen
+	 */
+	val isOptional: Boolean
+)

src/main/kotlin/link/infra/packwiz/installer/target/CachedTargetValidation.kt

@@ -0,0 +1,95 @@
+package link.infra.packwiz.installer.target
+
+import java.io.IOException
+import java.nio.file.*
+import java.nio.file.attribute.BasicFileAttributes
+import kotlin.io.path.relativeTo
+
+data class CachedTargetStatus(val target: CachedTarget, var isValid: Boolean, var markDisabled: Boolean)
+
+fun validate(targets: List<CachedTarget>, baseDir: Path) = runCatching {
+	val results = targets.map {
+		CachedTargetStatus(it, isValid = false, markDisabled = false)
+	}
+	val tree = buildTree(results, baseDir)
+
+	// Efficient file exists checking using directory listing, several orders of magnitude faster than Files.exists calls
+	Files.walkFileTree(baseDir, setOf(FileVisitOption.FOLLOW_LINKS), Int.MAX_VALUE, object : FileVisitor<Path> {
+		var currentNode: PathNode<CachedTargetStatus> = tree
+
+		override fun preVisitDirectory(dir: Path?, attrs: BasicFileAttributes?): FileVisitResult {
+			if (dir == null) {
+				return FileVisitResult.SKIP_SUBTREE
+			}
+			val subdirNode = currentNode.subdirs[dir.getName(dir.nameCount - 1)]
+			return if (subdirNode != null) {
+				currentNode = subdirNode
+				FileVisitResult.CONTINUE
+			} else {
+				FileVisitResult.SKIP_SUBTREE
+			}
+		}
+
+		override fun visitFile(file: Path?, attrs: BasicFileAttributes?): FileVisitResult {
+			if (file == null) {
+				return FileVisitResult.CONTINUE
+			}
+			// TODO: these are relative paths to baseDir
+			// TODO: strip the .disabled for lookup
+			val target = currentNode.files[file.getName(file.nameCount - 1)]
+			if (target != null) {
+				val disabledFile = file.endsWith(".disabled")
+				// If a .disabled file and the actual file both exist, mark as invalid if the target is disabled
+				if ((disabledFile )) {
+
+				}
+			}
+			return FileVisitResult.CONTINUE
+		}
+
+		@Throws(IOException::class)
+		override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult {
+			if (exc != null) {
+				throw exc
+			}
+			throw IOException("visitFileFailed called with no exception")
+		}
+
+		@Throws(IOException::class)
+		override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult {
+			if (exc != null) {
+				throw exc
+			} else {
+				val parent = currentNode.parent
+				if (parent != null) {
+					currentNode = parent
+				} else {
+					throw IOException("Invalid visitor tree structure")
+				}
+				return FileVisitResult.CONTINUE
+			}
+		}
+	})
+
+	results
+}
+
+fun buildTree(targets: List<CachedTargetStatus>, baseDir: Path): PathNode<CachedTargetStatus> {
+	val root = PathNode<CachedTargetStatus>()
+	for (target in targets) {
+		val relPath = target.target.cachedLocation.relativeTo(baseDir)
+		var node = root
+		// Traverse all the directory components, except for the last one
+		for (i in 0 until (relPath.nameCount - 1)) {
+			node = node.createSubdir(relPath.getName(i))
+		}
+		node.files[relPath.getName(relPath.nameCount - 1)] = target
+	}
+	return root
+}
+
+data class PathNode<T>(val subdirs: MutableMap<Path, PathNode<T>>, val files: MutableMap<Path, T>, val parent: PathNode<T>?) {
+	constructor() : this(mutableMapOf(), mutableMapOf(), null)
+
+	fun createSubdir(nextComponent: Path) = subdirs.getOrPut(nextComponent, { PathNode(mutableMapOf(), mutableMapOf(), this) })
+}

src/main/kotlin/link/infra/packwiz/installer/target/Side.kt

@@ -0,0 +1,54 @@
+package link.infra.packwiz.installer.target
+
+import com.google.gson.annotations.SerializedName
+
+enum class Side {
+	@SerializedName("client")
+	CLIENT("client"),
+	@SerializedName("server")
+	SERVER("server"),
+	@SerializedName("both")
+	@Suppress("unused")
+	BOTH("both", arrayOf(CLIENT, SERVER));
+
+	private val sideName: String
+	private val depSides: Array<Side>?
+
+	constructor(sideName: String) {
+		this.sideName = sideName.toLowerCase()
+		depSides = null
+	}
+
+	constructor(sideName: String, depSides: Array<Side>) {
+		this.sideName = sideName.toLowerCase()
+		this.depSides = depSides
+	}
+
+	override fun toString() = sideName
+
+	fun hasSide(tSide: Side): Boolean {
+		if (this == tSide) {
+			return true
+		}
+		if (depSides != null) {
+			for (depSide in depSides) {
+				if (depSide == tSide) {
+					return true
+				}
+			}
+		}
+		return false
+	}
+
+	companion object {
+		fun from(name: String): Side? {
+			val lowerName = name.toLowerCase()
+			for (side in values()) {
+				if (side.sideName == lowerName) {
+					return side
+				}
+			}
+			return null
+		}
+	}
+}

src/main/kotlin/link/infra/packwiz/installer/target/Target.kt

@@ -0,0 +1,32 @@
+package link.infra.packwiz.installer.target
+
+import link.infra.packwiz.installer.metadata.SpaceSafeURI
+import link.infra.packwiz.installer.metadata.hash.Hash
+import java.nio.file.Path
+
+data class Target(
+	/**
+	 * The name that uniquely identifies this target.
+	 * Often equal to the name of the metadata file for this target, and can be displayed to the user in progress UI.
+	 */
+	val name: String,
+	/**
+	 * An optional user-friendly name.
+	 */
+	val userFriendlyName: String?,
+	val src: SpaceSafeURI,
+	val dest: Path,
+	val hash: Hash,
+	val side: Side,
+	val optional: Boolean,
+	val optionalDefaultValue: Boolean,
+	val optionalDescription: String,
+	/**
+	 * If this is true, don't update a target when the file already exists.
+	 */
+	val noOverwrite: Boolean
+) {
+	fun Iterable<Target>.filterForSide(side: Side) = this.filter {
+		it.side.hasSide(side)
+	}
+}