Index fixes and performance improvements (fixes #223, #224)

- Fixed creation of duplicate index entries when importing from CurseForge (fixes #224)
- Automatically remove duplicates in index
- Fixed `packwiz serve` with a custom `--pack-root` argument (fixes #223)
- Fixed `packwiz serve` with a custom index.toml location
- Cleaned up internal serving code, added comments and better errors
- Refactored path handling code
- Improved refreshing/exporting performance
- Factored out duplicated exporting logic
- Replaced GetAllMods calls with cleaner LoadAllMods calls and made the former private
- Improved variable names in update command
- Improved handling of aliassed files
- Changed CheckUpdate to take references to metadata
- Removed the ability to use an absolute path to the index file (that probably didn't work anyway)
- Behaviour change: order of entries in exported files may be random

cmd/serve.go

@@ -48,8 +48,8 @@ var serveCmd = &cobra.Command{
 				fmt.Println(err)
 				os.Exit(1)
 			}
-			indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File))
-			indexDir := filepath.Dir(indexPath)
+			packServeDir := filepath.Dir(viper.GetString("pack-file"))
+			packFileName := filepath.Base(viper.GetString("pack-file"))
 
 			t, err := template.New("index-page").Parse(indexPage)
 			if err != nil {
@@ -75,103 +75,68 @@ var serveCmd = &cobra.Command{
 					return
 				}
 
+				// Relative to pack.toml
 				urlPath := strings.TrimPrefix(path.Clean("/"+strings.TrimPrefix(req.URL.Path, "/")), "/")
-				indexRelPath, err := filepath.Rel(indexDir, filepath.FromSlash(urlPath))
+				// Convert to absolute
+				destPath := filepath.Join(packServeDir, filepath.FromSlash(urlPath))
+				// Relativisation needs to be done using filepath, as path doesn't have Rel!
+				// (now using index util function)
+				// Relative to index.toml ("pack root")
+				indexRelPath, err := index.RelIndexPath(destPath)
 				if err != nil {
-					fmt.Println(err)
+					fmt.Println("Failed to parse path", err)
 					return
 				}
-				indexRelPathSlash := path.Clean(filepath.ToSlash(indexRelPath))
-				var destPath string
 
-				found := false
-				if urlPath == filepath.ToSlash(indexPath) {
-					found = true
-					destPath = indexPath
+				if urlPath == path.Clean(pack.Index.File) {
 					// Must be done here, to ensure all paths gain the lock at some point
 					refreshMutex.RLock()
-				} else if urlPath == filepath.ToSlash(viper.GetString("pack-file")) {
-					found = true
+				} else if urlPath == packFileName { // Only need to compare name - already relative to pack.toml
 					if viper.GetBool("serve.refresh") {
 						// Get write lock, to do a refresh
 						refreshMutex.Lock()
 						// Reload pack and index (might have changed on disk)
-						pack, err = core.LoadPack()
+						err = doServeRefresh(&pack, &index)
 						if err != nil {
-							fmt.Println(err)
+							fmt.Println("Failed to refresh pack", err)
 							return
 						}
-						index, err = pack.LoadIndex()
-						if err != nil {
-							fmt.Println(err)
-							return
-						}
-						err = index.Refresh()
-						if err != nil {
-							fmt.Println(err)
-							return
-						}
-						err = index.Write()
-						if err != nil {
-							fmt.Println(err)
-							return
-						}
-						err = pack.UpdateIndexHash()
-						if err != nil {
-							fmt.Println(err)
-							return
-						}
-						err = pack.Write()
-						if err != nil {
-							fmt.Println(err)
-							return
-						}
-						fmt.Println("Index refreshed!")
 
 						// Downgrade to a read lock
 						refreshMutex.Unlock()
 					}
 					refreshMutex.RLock()
-					destPath = viper.GetString("pack-file")
 				} else {
 					refreshMutex.RLock()
 					// Only allow indexed files
-					for _, v := range index.Files {
-						if indexRelPathSlash == v.File {
-							found = true
-							break
-						}
-					}
-					if found {
-						destPath = filepath.FromSlash(urlPath)
-					}
-				}
-				defer refreshMutex.RUnlock()
-				if found {
-					f, err := os.Open(destPath)
-					if err != nil {
-						fmt.Printf("Error reading file \"%s\": %s\n", destPath, err)
+					if _, found := index.Files[indexRelPath]; !found {
+						fmt.Printf("File not found: %s\n", destPath)
+						refreshMutex.RUnlock()
 						w.WriteHeader(404)
 						_, _ = w.Write([]byte("File not found"))
 						return
 					}
-					_, err = io.Copy(w, f)
-					err2 := f.Close()
-					if err == nil {
-						err = err2
-					}
-					if err != nil {
-						fmt.Printf("Error reading file \"%s\": %s\n", destPath, err)
-						w.WriteHeader(500)
-						_, _ = w.Write([]byte("Failed to read file"))
-						return
-					}
-				} else {
-					fmt.Printf("File not found: %s\n", destPath)
+				}
+				defer refreshMutex.RUnlock()
+
+				f, err := os.Open(destPath)
+				if err != nil {
+					fmt.Printf("Error reading file \"%s\": %s\n", destPath, err)
 					w.WriteHeader(404)
 					_, _ = w.Write([]byte("File not found"))
 					return
 				}
+				_, err = io.Copy(w, f)
+				err2 := f.Close()
+				if err == nil {
+					err = err2
+				}
+				if err != nil {
+					fmt.Printf("Error reading file \"%s\": %s\n", destPath, err)
+					w.WriteHeader(500)
+					_, _ = w.Write([]byte("Failed to read file"))
+					return
+				}
 			})
 		}
 
@@ -184,6 +149,37 @@ var serveCmd = &cobra.Command{
 	},
 }
 
+func doServeRefresh(pack *core.Pack, index *core.Index) error {
+	var err error
+	*pack, err = core.LoadPack()
+	if err != nil {
+		return err
+	}
+	*index, err = pack.LoadIndex()
+	if err != nil {
+		return err
+	}
+	err = index.Refresh()
+	if err != nil {
+		return err
+	}
+	err = index.Write()
+	if err != nil {
+		return err
+	}
+	err = pack.UpdateIndexHash()
+	if err != nil {
+		return err
+	}
+	err = pack.Write()
+	if err != nil {
+		return err
+	}
+	fmt.Println("Index refreshed!")
+
+	return nil
+}
+
 func init() {
 	rootCmd.AddCommand(serveCmd)
 

cmd/update.go

@@ -33,27 +33,26 @@ var UpdateCmd = &cobra.Command{
 
 		var singleUpdatedName string
 		if viper.GetBool("update.all") {
-			updaterMap := make(map[string][]core.Mod)
+			filesWithUpdater := make(map[string][]*core.Mod)
 			fmt.Println("Reading metadata files...")
-			for _, v := range index.GetAllMods() {
-				modData, err := core.LoadMod(v)
-				if err != nil {
-					fmt.Printf("Error reading metadata file: %s\n", err.Error())
-					continue
-				}
-
+			mods, err := index.LoadAllMods()
+			if err != nil {
+				fmt.Printf("Failed to update all files: %v\n", err)
+				os.Exit(1)
+			}
+			for _, modData := range mods {
 				updaterFound := false
 				for k := range modData.Update {
-					slice, ok := updaterMap[k]
+					slice, ok := filesWithUpdater[k]
 					if !ok {
 						_, ok = core.Updaters[k]
 						if !ok {
 							continue
 						}
-						slice = []core.Mod{}
+						slice = []*core.Mod{}
 					}
 					updaterFound = true
-					updaterMap[k] = append(slice, modData)
+					filesWithUpdater[k] = append(slice, modData)
 				}
 				if !updaterFound {
 					fmt.Printf("A supported update system for \"%s\" cannot be found.\n", modData.Name)
@@ -62,9 +61,9 @@ var UpdateCmd = &cobra.Command{
 
 			fmt.Println("Checking for updates...")
 			updatesFound := false
-			updaterPointerMap := make(map[string][]*core.Mod)
+			updatableFiles := make(map[string][]*core.Mod)
 			updaterCachedStateMap := make(map[string][]interface{})
-			for k, v := range updaterMap {
+			for k, v := range filesWithUpdater {
 				checks, err := core.Updaters[k].CheckUpdate(v, pack)
 				if err != nil {
 					// TODO: do we return err code 1?
@@ -83,7 +82,7 @@ var UpdateCmd = &cobra.Command{
 							updatesFound = true
 						}
 						fmt.Printf("%s: %s\n", v[i].Name, check.UpdateString)
-						updaterPointerMap[k] = append(updaterPointerMap[k], &v[i])
+						updatableFiles[k] = append(updatableFiles[k], v[i])
 						updaterCachedStateMap[k] = append(updaterCachedStateMap[k], check.CachedState)
 					}
 				}
@@ -99,7 +98,7 @@ var UpdateCmd = &cobra.Command{
 				return
 			}
 
-			for k, v := range updaterPointerMap {
+			for k, v := range updatableFiles {
 				err := core.Updaters[k].DoUpdate(v, updaterCachedStateMap[k])
 				if err != nil {
 					// TODO: do we return err code 1?
@@ -143,7 +142,7 @@ var UpdateCmd = &cobra.Command{
 				}
 				updaterFound = true
 
-				check, err := updater.CheckUpdate([]core.Mod{modData}, pack)
+				check, err := updater.CheckUpdate([]*core.Mod{&modData}, pack)
 				if err != nil {
 					fmt.Println(err)
 					os.Exit(1)