Completed download implementation for Modrinth export

cmdshared/downloadutil.go

@@ -1,8 +1,10 @@
 package cmdshared
 
 import (
+	"archive/zip"
 	"fmt"
 	"github.com/packwiz/packwiz/core"
+	"io"
 	"os"
 	"path/filepath"
 )
@@ -26,3 +28,48 @@ func ListManualDownloads(session core.DownloadSession) {
 		os.Exit(1)
 	}
 }
+
+func AddToZip(dl core.CompletedDownload, exp *zip.Writer, dir string, indexPath string) bool {
+	if dl.Error != nil {
+		fmt.Printf("Download of %s (%s) failed: %v\n", dl.Mod.Name, dl.Mod.FileName, dl.Error)
+		return false
+	}
+	for warning := range dl.Warnings {
+		fmt.Printf("Warning for %s (%s): %v\n", dl.Mod.Name, dl.Mod.FileName, warning)
+	}
+
+	path, err := filepath.Rel(filepath.Dir(indexPath), dl.Mod.GetDestFilePath())
+	if err != nil {
+		fmt.Printf("Error resolving mod file: %v\n", err)
+		return false
+	}
+	modFile, err := exp.Create(filepath.ToSlash(filepath.Join(dir, path)))
+	if err != nil {
+		fmt.Printf("Error creating mod file %s: %v\n", path, err)
+		return false
+	}
+	_, err = io.Copy(modFile, dl.File)
+	if err != nil {
+		fmt.Printf("Error copying file %s: %v\n", path, err)
+		return false
+	}
+	err = dl.File.Close()
+	if err != nil {
+		fmt.Printf("Error closing file %s: %v\n", path, err)
+		return false
+	}
+
+	fmt.Printf("%s (%s) added to zip\n", dl.Mod.Name, dl.Mod.FileName)
+	return true
+}
+
+func PrintDisclaimer(isCf bool) {
+	fmt.Println("Disclaimer: you are responsible for ensuring you comply with ALL the licenses, or obtain appropriate permissions, for the files \"added to zip\" below")
+	if isCf {
+		fmt.Println("Note that mods bundled within a CurseForge pack must be in the Approved Non-CurseForge Mods list")
+		fmt.Println("packwiz is currently unable to match metadata between mod sites - if any of these are available from CurseForge you should change them to use CurseForge metadata (e.g. by reinstalling them using the cf commands)")
+	} else {
+		fmt.Println("packwiz is currently unable to match metadata between mod sites - if any of these are available from Modrinth you should change them to use Modrinth metadata (e.g. by reinstalling them using the mr commands)")
+	}
+	fmt.Println()
+}

core/index.go

@@ -382,6 +382,9 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error {
 		hashFormat = in.HashFormat
 	}
 	src, err := os.Open(in.GetFilePath(f))
+	defer func(src *os.File) {
+		_ = src.Close()
+	}(src)
 	if err != nil {
 		return err
 	}

curseforge/export.go

@@ -9,7 +9,6 @@ import (
 	"github.com/packwiz/packwiz/curseforge/packinterop"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"io"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -129,10 +128,7 @@ var exportCmd = &cobra.Command{
 		// Download external files and save directly into the zip
 		if len(nonCfMods) > 0 {
 			fmt.Printf("Retrieving %v external files to store in the modpack zip...\n", len(nonCfMods))
-			fmt.Println("Disclaimer: you are responsible for ensuring you comply with ALL the licenses, or obtain appropriate permissions, for the files listed below")
+			cmdshared.PrintDisclaimer(true)
-			fmt.Println("Note that mods bundled within a CurseForge pack must be in the Approved Non-CurseForge Mods list")
-			fmt.Println("packwiz is currently unable to match metadata between mod sites - if any of these are available from CurseForge you should change them to use CurseForge metadata (e.g. by reinstalling them using the cf commands)")
-			fmt.Println()
 
 			session, err := core.CreateDownloadSession(nonCfMods, []string{})
 			if err != nil {
@@ -143,31 +139,7 @@ var exportCmd = &cobra.Command{
 			cmdshared.ListManualDownloads(session)
 
 			for dl := range session.StartDownloads() {
-				if dl.Error != nil {
+				_ = cmdshared.AddToZip(dl, exp, "overrides", indexPath)
-					fmt.Printf("Download of %s (%s) failed: %v\n", dl.Mod.Name, dl.Mod.FileName, dl.Error)
-					continue
-				}
-				for warning := range dl.Warnings {
-					fmt.Printf("Warning for %s (%s): %v\n", dl.Mod.Name, dl.Mod.FileName, warning)
-				}
-
-				path, err := filepath.Rel(filepath.Dir(indexPath), dl.Mod.GetDestFilePath())
-				if err != nil {
-					fmt.Printf("Error resolving mod file: %v\n", err)
-					continue
-				}
-				modFile, err := exp.Create(filepath.ToSlash(filepath.Join("overrides", path)))
-				if err != nil {
-					fmt.Printf("Error creating mod file %s: %v\n", path, err)
-					continue
-				}
-				_, err = io.Copy(modFile, dl.File)
-				if err != nil {
-					fmt.Printf("Error copying file %s: %v\n", path, err)
-					continue
-				}
-
-				fmt.Printf("%s (%s) added to zip\n", dl.Mod.Name, dl.Mod.FileName)
 			}
 
 			err = session.SaveIndex()
@@ -189,7 +161,7 @@ var exportCmd = &cobra.Command{
 		if err != nil {
 			_ = exp.Close()
 			_ = expFile.Close()
-			fmt.Println("Error creating manifest: " + err.Error())
+			fmt.Println("Error writing manifest: " + err.Error())
 			os.Exit(1)
 		}
 

curseforge/request.go

@@ -12,8 +12,6 @@ import (
 	"time"
 )
 
-// TODO: update everything for no URL and download mode "metadata:curseforge"
-
 const cfApiServer = "api.curseforge.com"
 
 // If you fork/derive from packwiz, I request that you obtain your own API key.

modrinth/export.go

@@ -4,7 +4,10 @@ import (
 	"archive/zip"
 	"encoding/json"
 	"fmt"
+	"github.com/packwiz/packwiz/cmdshared"
 	"github.com/spf13/viper"
+	"golang.org/x/exp/slices"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -80,84 +83,104 @@ var exportCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		// TODO: finish updating to use download session
+		fmt.Printf("Retrieving %v external files...\n", len(mods))
-		fmt.Println("Retrieving external mods...")
-		session, err := core.CreateDownloadSession(mods, []string{"sha1", "sha512", "length-bytes"})
-		_ = session
 
-		modsHashes := make([]map[string]string, len(mods))
+		for _, mod := range mods {
-		for i, mod := range mods {
+			if !canBeIncludedDirectly(mod) {
-			modsHashes[i], err = mod.GetHashes([]string{"sha1", "sha512", "length-bytes"})
+				cmdshared.PrintDisclaimer(false)
-			if err != nil {
+				break
-				fmt.Printf("Error downloading mod file %s: %s\n", mod.Download.URL, err.Error())
-				continue
 			}
-			fmt.Printf("Retrieved hashes for %s successfully\n", mod.Download.URL)
 		}
 
-		manifestFile, err := exp.Create("modrinth.index.json")
+		session, err := core.CreateDownloadSession(mods, []string{"sha1", "sha512", "length-bytes"})
 		if err != nil {
-			_ = exp.Close()
+			fmt.Printf("Error retrieving external files: %v\n", err)
-			_ = expFile.Close()
-			fmt.Println("Error creating manifest: " + err.Error())
 			os.Exit(1)
 		}
 
-		manifestFiles := make([]PackFile, len(mods))
+		cmdshared.ListManualDownloads(session)
-		for i, mod := range mods {
-			pathForward, err := filepath.Rel(filepath.Dir(indexPath), mod.GetDestFilePath())
-			if err != nil {
-				fmt.Printf("Error resolving mod file: %s\n", err.Error())
-				// TODO: exit(1)?
-				continue
-			}
 
-			path := filepath.ToSlash(pathForward)
+		manifestFiles := make([]PackFile, 0)
+		for dl := range session.StartDownloads() {
+			if canBeIncludedDirectly(dl.Mod) {
+				if dl.Error != nil {
+					fmt.Printf("Download of %s (%s) failed: %v\n", dl.Mod.Name, dl.Mod.FileName, dl.Error)
+					continue
+				}
+				for warning := range dl.Warnings {
+					fmt.Printf("Warning for %s (%s): %v\n", dl.Mod.Name, dl.Mod.FileName, warning)
+				}
 
-			hashes := make(map[string]string)
+				pathForward, err := filepath.Rel(filepath.Dir(indexPath), dl.Mod.GetDestFilePath())
-			hashes["sha1"] = modsHashes[i]["sha1"]
+				if err != nil {
-			hashes["sha512"] = modsHashes[i]["sha512"]
+					fmt.Printf("Error resolving mod file: %s\n", err.Error())
-			fileSize, err := strconv.ParseUint(modsHashes[i]["length-bytes"], 10, 64)
+					// TODO: exit(1)?
-			if err != nil {
+					continue
-				panic(err)
+				}
-			}
 
-			// Create env options based on configured optional/side
+				path := filepath.ToSlash(pathForward)
-			var envInstalled string
+
-			if mod.Option != nil && mod.Option.Optional {
+				hashes := make(map[string]string)
-				envInstalled = "optional"
+				hashes["sha1"] = dl.Hashes["sha1"]
+				hashes["sha512"] = dl.Hashes["sha512"]
+				fileSize, err := strconv.ParseUint(dl.Hashes["length-bytes"], 10, 64)
+				if err != nil {
+					panic(err)
+				}
+
+				// Create env options based on configured optional/side
+				var envInstalled string
+				if dl.Mod.Option != nil && dl.Mod.Option.Optional {
+					envInstalled = "optional"
+				} else {
+					envInstalled = "required"
+				}
+				var clientEnv, serverEnv string
+				if dl.Mod.Side == core.UniversalSide {
+					clientEnv = envInstalled
+					serverEnv = envInstalled
+				} else if dl.Mod.Side == core.ClientSide {
+					clientEnv = envInstalled
+					serverEnv = "unsupported"
+				} else if dl.Mod.Side == core.ServerSide {
+					clientEnv = "unsupported"
+					serverEnv = envInstalled
+				}
+
+				// Modrinth URLs must be RFC3986
+				u, err := core.ReencodeURL(dl.Mod.Download.URL)
+				if err != nil {
+					fmt.Printf("Error re-encoding mod URL: %s\n", err.Error())
+					u = dl.Mod.Download.URL
+				}
+
+				manifestFiles = append(manifestFiles, PackFile{
+					Path:   path,
+					Hashes: hashes,
+					Env: &struct {
+						Client string `json:"client"`
+						Server string `json:"server"`
+					}{Client: clientEnv, Server: serverEnv},
+					Downloads: []string{u},
+					FileSize:  uint32(fileSize),
+				})
+
+				fmt.Printf("%s (%s) added to manifest\n", dl.Mod.Name, dl.Mod.FileName)
 			} else {
-				envInstalled = "required"
+				if dl.Mod.Side == core.ClientSide {
-			}
+					_ = cmdshared.AddToZip(dl, exp, "client-overrides", indexPath)
-			var clientEnv, serverEnv string
+				} else if dl.Mod.Side == core.ServerSide {
-			if mod.Side == core.UniversalSide {
+					_ = cmdshared.AddToZip(dl, exp, "server-overrides", indexPath)
-				clientEnv = envInstalled
+				} else {
-				serverEnv = envInstalled
+					_ = cmdshared.AddToZip(dl, exp, "overrides", indexPath)
-			} else if mod.Side == core.ClientSide {
+				}
-				clientEnv = envInstalled
-				serverEnv = "unsupported"
-			} else if mod.Side == core.ServerSide {
-				clientEnv = "unsupported"
-				serverEnv = envInstalled
 			}
+		}
 
-			// Modrinth URLs must be RFC3986
+		err = session.SaveIndex()
-			u, err := core.ReencodeURL(mod.Download.URL)
+		if err != nil {
-			if err != nil {
+			fmt.Printf("Error saving cache index: %v\n", err)
-				fmt.Printf("Error re-encoding mod URL: %s\n", err.Error())
+			os.Exit(1)
-				u = mod.Download.URL
-			}
-
-			manifestFiles[i] = PackFile{
-				Path:   path,
-				Hashes: hashes,
-				Env: &struct {
-					Client string `json:"client"`
-					Server string `json:"server"`
-				}{Client: clientEnv, Server: serverEnv},
-				Downloads: []string{u},
-				FileSize:  uint32(fileSize),
-			}
 		}
 
 		dependencies := make(map[string]string)
@@ -190,6 +213,14 @@ var exportCmd = &cobra.Command{
 			fmt.Println("Warning: pack.toml version field must not be empty to create a valid Modrinth pack")
 		}
 
+		manifestFile, err := exp.Create("modrinth.index.json")
+		if err != nil {
+			_ = exp.Close()
+			_ = expFile.Close()
+			fmt.Println("Error creating manifest: " + err.Error())
+			os.Exit(1)
+		}
+
 		w := json.NewEncoder(manifestFile)
 		w.SetIndent("", "    ") // Documentation uses 4 spaces
 		err = w.Encode(manifest)
@@ -224,42 +255,6 @@ var exportCmd = &cobra.Command{
 			}
 		}
 
-		// TODO: get rid of this, do whitelist checks elsewhere
-
-		if len(unwhitelistedMods) > 0 {
-			fmt.Println("Downloading unwhitelisted mods...")
-		}
-		for _, v := range unwhitelistedMods {
-			pathRel, err := filepath.Rel(filepath.Dir(indexPath), v.GetDestFilePath())
-			if err != nil {
-				fmt.Printf("Error resolving mod file: %s\n", err.Error())
-				// TODO: exit(1)?
-				continue
-			}
-			var path string
-			if v.Side == core.ClientSide {
-				path = filepath.ToSlash(filepath.Join("client-overrides", pathRel))
-			} else if v.Side == core.ServerSide {
-				path = filepath.ToSlash(filepath.Join("server-overrides", pathRel))
-			} else {
-				path = filepath.ToSlash(filepath.Join("overrides", pathRel))
-			}
-
-			file, err := exp.Create(path)
-			if err != nil {
-				fmt.Printf("Error creating file: %s\n", err.Error())
-				// TODO: exit(1)?
-				continue
-			}
-			err = v.DownloadFile(file)
-			if err != nil {
-				fmt.Printf("Error downloading file: %s\n", err.Error())
-				// TODO: exit(1)?
-				continue
-			}
-			fmt.Printf("Downloaded %v successfully\n", path)
-		}
-
 		err = exp.Close()
 		if err != nil {
 			fmt.Println("Error writing export file: " + err.Error())
@@ -275,22 +270,25 @@ var exportCmd = &cobra.Command{
 	},
 }
 
-// TODO: update whitelist
 var whitelistedHosts = []string{
 	"cdn.modrinth.com",
 	"edge.forgecdn.net",
+	"media.forgecdn.net",
 	"github.com",
 	"raw.githubusercontent.com",
 }
 
-//modUrl, err := url.Parse(modData.Download.URL)
+func canBeIncludedDirectly(mod *core.Mod) bool {
-//if err == nil {
+	if mod.Download.Mode == "url" || mod.Download.Mode == "" {
-//if slices.Contains(whitelistedHosts, modUrl.Host) {
+		modUrl, err := url.Parse(mod.Download.URL)
-//mods = append(mods, modData)
+		if err == nil {
-//} else {
+			if slices.Contains(whitelistedHosts, modUrl.Host) {
-//unwhitelistedMods = append(unwhitelistedMods, modData)
+				return true
-//}
+			}
-//}
+		}
+	}
+	return false
+}
 
 func init() {
 	modrinthCmd.AddCommand(exportCmd)