diff --git a/cmdshared/downloadutil.go b/cmdshared/downloadutil.go new file mode 100644 index 0000000..fc66c6e --- /dev/null +++ b/cmdshared/downloadutil.go @@ -0,0 +1,32 @@ +package cmdshared + +import ( + "fmt" + "github.com/packwiz/packwiz/core" + "os" + "path/filepath" +) + +func ListManualDownloads(session core.DownloadSession) { + manualDownloads := session.GetManualDownloads() + if len(manualDownloads) > 0 { + fmt.Printf("Found %v manual downloads; these mods are unable to be downloaded by packwiz (due to API limitations) and must be manually downloaded:\n", + len(manualDownloads)) + for _, dl := range manualDownloads { + fmt.Printf("%s (%s) from %s\n", dl.Name, dl.FileName, dl.URL) + } + cacheDir, err := core.GetPackwizCache() + if err != nil { + fmt.Printf("Error locating cache folder: %v", err) + os.Exit(1) + } + err = os.MkdirAll(filepath.Join(cacheDir, core.DownloadCacheInFolder), 0755) + if err != nil { + fmt.Printf("Error creating cache in folder: %v", err) + os.Exit(1) + } + fmt.Printf("Once you have done so, place these files in %s and re-run this command.\n", + filepath.Join(cacheDir, core.DownloadCacheInFolder)) + os.Exit(1) + } +} diff --git a/core/download.go b/core/download.go index 290b57d..66c3bbf 100644 --- a/core/download.go +++ b/core/download.go @@ -13,6 +13,8 @@ import ( "strings" ) +const DownloadCacheInFolder = "in" + type DownloadSession interface { GetManualDownloads() []ManualDownload StartDownloads() chan CompletedDownload @@ -20,9 +22,9 @@ type DownloadSession interface { } type CompletedDownload struct { - File *os.File - DestFilePath string - Hashes map[string]string + File *os.File + Mod *Mod + Hashes map[string]string // Error indicates if/why downloading this file failed Error error // Warnings indicates messages to show to the user regarding this file (download was successful, but had a problem) @@ -39,7 +41,7 @@ type downloadSessionInternal struct { type downloadTask struct { metaDownloaderData MetaDownloaderData - destFilePath string + mod *Mod url string hashFormat string hash string @@ -52,11 +54,23 @@ func (d *downloadSessionInternal) GetManualDownloads() []ManualDownload { func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload { downloads := make(chan CompletedDownload) - for _, task := range d.downloadTasks { - // Get handle for mod - cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash) - if cacheHandle != nil { - download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.destFilePath) + go func() { + for _, task := range d.downloadTasks { + // Get handle for mod + cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash) + if cacheHandle != nil { + download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.mod) + if err != nil { + downloads <- CompletedDownload{ + Error: err, + } + } else { + downloads <- download + } + continue + } + + download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex) if err != nil { downloads <- CompletedDownload{ Error: err, @@ -64,18 +78,9 @@ func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload { } else { downloads <- download } - continue } - - download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex) - if err != nil { - downloads <- CompletedDownload{ - Error: err, - } - } else { - downloads <- download - } - } + close(downloads) + }() return downloads } @@ -91,7 +96,7 @@ func (d *downloadSessionInternal) SaveIndex() error { return nil } -func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, destFilePath string) (CompletedDownload, error) { +func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, mod *Mod) (CompletedDownload, error) { // Already stored; try using it! file, err := cacheHandle.Open() if err == nil { @@ -111,9 +116,9 @@ func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, d } return CompletedDownload{ - File: file, - DestFilePath: destFilePath, - Hashes: cacheHandle.Hashes, + File: file, + Mod: mod, + Hashes: cacheHandle.Hashes, }, nil } else { return CompletedDownload{}, fmt.Errorf("failed to read file %s from cache: %w", cacheHandle.Path(), err) @@ -151,7 +156,7 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st err = teeHashes(hashesToObtain, hashes, tempFile, data) _ = data.Close() if err != nil { - return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.destFilePath, err) + return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.mod.Name, err) } } @@ -180,14 +185,14 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st } return CompletedDownload{ - File: file, - DestFilePath: task.destFilePath, - Hashes: hashes, - Warnings: warnings, + File: file, + Mod: task.mod, + Hashes: hashes, + Warnings: warnings, }, nil } -func selectPreferredHash(hashes map[string]string) (currHash string, currHashFormat string) { +func selectPreferredHash(hashes map[string]string) (currHashFormat string, currHash string) { for _, hashFormat := range preferredHashList { if hash, ok := hashes[hashFormat]; ok { currHashFormat = hashFormat @@ -226,7 +231,7 @@ func teeHashes(hashesToObtain []string, hashes map[string]string, return fmt.Errorf("failed to get hash format %s", validateHashFormat) } hashers := make(map[string]HashStringer, len(hashesToObtain)) - allWriters := make([]io.Writer, len(hashers)) + allWriters := make([]io.Writer, len(hashesToObtain)) for i, v := range hashesToObtain { hashers[v], err = GetHashImpl(v) if err != nil { @@ -340,11 +345,11 @@ func (h *CacheIndexHandle) CreateFromTemp(temp *os.File) (*os.File, error) { if err != nil { return nil, err } - err = os.Rename(temp.Name(), h.Path()) + err = temp.Close() if err != nil { return nil, err } - err = temp.Close() + err = os.Rename(temp.Name(), h.Path()) if err != nil { return nil, err } @@ -425,12 +430,12 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio // Get necessary metadata for all files for _, mod := range mods { - if mod.Download.Mode == "url" { + if mod.Download.Mode == "url" || mod.Download.Mode == "" { downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{ - destFilePath: mod.GetDestFilePath(), - url: mod.Download.URL, - hashFormat: mod.Download.HashFormat, - hash: mod.Download.Hash, + mod: mod, + url: mod.Download.URL, + hashFormat: mod.Download.HashFormat, + hash: mod.Download.Hash, }) } else if strings.HasPrefix(mod.Download.Mode, "metadata:") { dlID := strings.TrimPrefix(mod.Download.Mode, "metadata:") @@ -452,10 +457,11 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio for i, v := range mods { isManual, manualDownload := meta[i].GetManualDownload() if isManual { + // TODO: lookup in index! downloadSession.manualDownloads = append(downloadSession.manualDownloads, manualDownload) } else { downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{ - destFilePath: v.GetDestFilePath(), + mod: v, metaDownloaderData: meta[i], hashFormat: v.Download.HashFormat, hash: v.Download.Hash, diff --git a/core/index.go b/core/index.go index c51eca6..7295d2a 100644 --- a/core/index.go +++ b/core/index.go @@ -2,6 +2,7 @@ package core import ( "errors" + "fmt" "io" "io/ioutil" "os" @@ -355,6 +356,20 @@ func (in Index) GetAllMods() []string { return list } +// LoadAllMods reads all metadata files into Mod structs +func (in Index) LoadAllMods() ([]*Mod, error) { + modPaths := in.GetAllMods() + mods := make([]*Mod, len(modPaths)) + for i, v := range modPaths { + modData, err := LoadMod(v) + if err != nil { + return nil, fmt.Errorf("failed to read metadata file %s: %w", v, err) + } + mods[i] = &modData + } + return mods, nil +} + // GetFilePath attempts to get the path of the destination index file as it is stored on disk func (in Index) GetFilePath(f IndexFile) string { return filepath.Join(filepath.Dir(in.indexFile), filepath.FromSlash(f.File)) diff --git a/core/interfaces.go b/core/interfaces.go index 2587c81..ccfd71d 100644 --- a/core/interfaces.go +++ b/core/interfaces.go @@ -51,6 +51,5 @@ type MetaDownloaderData interface { type ManualDownload struct { Name string FileName string - DestPath string URL string } diff --git a/core/mod.go b/core/mod.go index 63c9348..6ce570a 100644 --- a/core/mod.go +++ b/core/mod.go @@ -2,15 +2,10 @@ package core import ( "errors" - "fmt" - "golang.org/x/exp/slices" + "github.com/BurntSushi/toml" "io" - "net/http" "os" "path/filepath" - "strconv" - - "github.com/BurntSushi/toml" ) // Mod stores metadata about a mod. This is written to a TOML file for each mod. @@ -134,106 +129,3 @@ func (m Mod) GetFilePath() string { func (m Mod) GetDestFilePath() string { return filepath.Join(filepath.Dir(m.metaFile), filepath.FromSlash(m.FileName)) } - -// DownloadFile attempts to resolve and download the file -func (m Mod) DownloadFile(dest io.Writer) error { - // TODO: check mode - resp, err := http.Get(m.Download.URL) - // TODO: content type, user-agent? - if err != nil { - return err - } - if resp.StatusCode != 200 { - _ = resp.Body.Close() - return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode)) - } - h, err := GetHashImpl(m.Download.HashFormat) - if err != nil { - return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err) - } - - w := io.MultiWriter(h, dest) - _, err = io.Copy(w, resp.Body) - if err != nil { - return err - } - - calculatedHash := h.HashToString(h.Sum(nil)) - - // Check if the hash of the downloaded file matches the expected hash. - if calculatedHash != m.Download.Hash { - return fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash) - } - - return nil -} - -// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary -func (m Mod) GetHashes(hashes []string) (map[string]string, error) { - out := make(map[string]string) - - // Get the hash already stored TODO: store multiple (requires breaking pack change) - if m.Download.Hash != "" { - idx := slices.Index(hashes, m.Download.HashFormat) - if idx > -1 { - out[m.Download.HashFormat] = m.Download.Hash - // Remove hash from list to retrieve - hashes = slices.Delete(hashes, idx, idx+1) - } - } - - // Retrieve the remaining hashes - if len(hashes) > 0 { - // TODO: check mode - resp, err := http.Get(m.Download.URL) - // TODO: content type, user-agent? - if err != nil { - return nil, err - } - if resp.StatusCode != 200 { - _ = resp.Body.Close() - return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode)) - } - - // Special fast-path for file length only - if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 { - out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10) - _ = resp.Body.Close() - return out, nil - } - - mainHasher, err := GetHashImpl(m.Download.HashFormat) - if err != nil { - return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err) - } - - hashers := make([]HashStringer, len(hashes)) - allHashers := make([]io.Writer, len(hashers)) - for i, v := range hashes { - hashers[i], err = GetHashImpl(v) - if err != nil { - return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err) - } - allHashers[i] = hashers[i] - } - allHashers = append(allHashers, mainHasher) - - w := io.MultiWriter(allHashers...) - _, err = io.Copy(w, resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to download file: %w", err) - } - - calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil)) - - // Check if the hash of the downloaded file matches the expected hash - if calculatedHash != m.Download.Hash { - return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash) - } - - for i, v := range hashers { - out[hashes[i]] = v.HashToString(v.Sum(nil)) - } - } - return out, nil -} diff --git a/curseforge/curseforge.go b/curseforge/curseforge.go index 8dd927e..a33a765 100644 --- a/curseforge/curseforge.go +++ b/curseforge/curseforge.go @@ -194,6 +194,7 @@ func createModFile(modInfo modInfo, fileInfo modFileInfo, index *core.Index, opt Download: core.ModDownload{ HashFormat: hashFormat, Hash: hash, + Mode: "metadata:curseforge", }, Option: optional, Update: updateMap, @@ -422,6 +423,7 @@ func (u cfUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { v.Download = core.ModDownload{ HashFormat: hashFormat, Hash: hash, + Mode: "metadata:curseforge", } v.Update["curseforge"]["project-id"] = modState.ID @@ -481,9 +483,10 @@ func (c cfDownloader) GetFilesMetadata(mods []*core.Mod) ([]core.MetaDownloaderD meta.websiteUrl = meta.websiteUrl + "/files/" + strconv.Itoa(fileInfo.ID) meta.fileName = fileInfo.FileName } - } - downloaderData[indexMap[modID]] = &cfDownloadMetadata{ - url: fileInfo.DownloadURL, + } else { + downloaderData[indexMap[modID]] = &cfDownloadMetadata{ + url: fileInfo.DownloadURL, + } } } diff --git a/curseforge/export.go b/curseforge/export.go index 39c11a2..32d655a 100644 --- a/curseforge/export.go +++ b/curseforge/export.go @@ -4,14 +4,15 @@ import ( "archive/zip" "bufio" "fmt" + "github.com/packwiz/packwiz/cmdshared" + "github.com/packwiz/packwiz/core" "github.com/packwiz/packwiz/curseforge/packinterop" + "github.com/spf13/cobra" "github.com/spf13/viper" + "io" "os" "path/filepath" "strconv" - - "github.com/packwiz/packwiz/core" - "github.com/spf13/cobra" ) // exportCmd represents the export command @@ -41,28 +42,33 @@ var exportCmd = &cobra.Command{ err = index.Refresh() if err != nil { fmt.Println(err) - return + os.Exit(1) } err = index.Write() if err != nil { fmt.Println(err) - return + os.Exit(1) } err = pack.UpdateIndexHash() if err != nil { fmt.Println(err) - return + os.Exit(1) } err = pack.Write() if err != nil { fmt.Println(err) - return + os.Exit(1) } // TODO: should index just expose indexPath itself, through a function? indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File)) - mods := loadMods(index) + fmt.Println("Reading external files...") + mods, err := index.LoadAllMods() + if err != nil { + fmt.Printf("Error reading file: %v\n", err) + os.Exit(1) + } i := 0 // Filter mods by side // TODO: opt-in optional disabled filtering? @@ -104,11 +110,13 @@ var exportCmd = &cobra.Command{ } cfFileRefs := make([]packinterop.AddonFileReference, 0, len(mods)) + nonCfMods := make([]*core.Mod, 0) for _, mod := range mods { projectRaw, ok := mod.GetParsedUpdateData("curseforge") // If the mod has curseforge metadata, add it to cfFileRefs - // TODO: how to handle files with CF metadata, but with different download path? - if ok { + // TODO: change back to use ok + _ = ok + if false { p := projectRaw.(cfUpdateData) cfFileRefs = append(cfFileRefs, packinterop.AddonFileReference{ ProjectID: p.ProjectID, @@ -116,26 +124,59 @@ var exportCmd = &cobra.Command{ OptionalDisabled: mod.Option != nil && mod.Option.Optional && !mod.Option.Default, }) } else { - // If the mod doesn't have the metadata, save it into the zip - path, err := filepath.Rel(filepath.Dir(indexPath), mod.GetDestFilePath()) + nonCfMods = append(nonCfMods, mod) + } + } + + // Download external files and save directly into the zip + if len(nonCfMods) > 0 { + fmt.Printf("Retrieving %v external files to store in the modpack zip...\n", len(nonCfMods)) + fmt.Println("Disclaimer: you are responsible for ensuring you comply with ALL the licenses, or obtain appropriate permissions, for the files listed below") + fmt.Println("Note that mods bundled within a CurseForge pack must be in the Approved Non-CurseForge Mods list") + fmt.Println("packwiz is currently unable to match metadata between mod sites - if any of these are available from CurseForge you should change them to use CurseForge metadata (e.g. by reinstalling them using the cf commands)") + fmt.Println() + + session, err := core.CreateDownloadSession(nonCfMods, []string{}) + if err != nil { + fmt.Printf("Error retrieving external files: %v\n", err) + os.Exit(1) + } + + cmdshared.ListManualDownloads(session) + + for dl := range session.StartDownloads() { + if dl.Error != nil { + // TODO: ensure name is populated + fmt.Printf("Download of %s (%s) failed: %v\n", dl.Mod.Name, dl.Mod.FileName, dl.Error) + continue + } + for warning := range dl.Warnings { + // TODO: get name + fmt.Printf("Download warning: %v\n", warning) + } + + path, err := filepath.Rel(filepath.Dir(indexPath), dl.Mod.GetDestFilePath()) if err != nil { - fmt.Printf("Error resolving mod file: %s\n", err.Error()) - // TODO: exit(1)? + fmt.Printf("Error resolving mod file: %v\n", err) continue } modFile, err := exp.Create(filepath.ToSlash(filepath.Join("overrides", path))) if err != nil { - fmt.Printf("Error creating mod file %s: %s\n", path, err.Error()) - // TODO: exit(1)? + fmt.Printf("Error creating mod file %s: %v\n", path, err) continue } - err = mod.DownloadFile(modFile) + _, err = io.Copy(modFile, dl.File) if err != nil { - fmt.Printf("Error downloading mod file %s: %s\n", path, err.Error()) - // TODO: exit(1)? + fmt.Printf("Error copying file %s: %v\n", path, err) continue } } + + err = session.SaveIndex() + if err != nil { + fmt.Printf("Error saving cache index: %v\n", err) + os.Exit(1) + } } manifestFile, err := exp.Create("manifest.json") @@ -203,7 +244,7 @@ var exportCmd = &cobra.Command{ }, } -func createModlist(zw *zip.Writer, mods []core.Mod) error { +func createModlist(zw *zip.Writer, mods []*core.Mod) error { modlistFile, err := zw.Create("modlist.html") if err != nil { return err @@ -239,25 +280,6 @@ func createModlist(zw *zip.Writer, mods []core.Mod) error { return w.Flush() } -func loadMods(index core.Index) []core.Mod { - modPaths := index.GetAllMods() - mods := make([]core.Mod, len(modPaths)) - i := 0 - fmt.Println("Reading mod files...") - for _, v := range modPaths { - modData, err := core.LoadMod(v) - if err != nil { - fmt.Printf("Error reading mod file %s: %s\n", v, err.Error()) - // TODO: exit(1)? - continue - } - - mods[i] = modData - i++ - } - return mods[:i] -} - func init() { curseforgeCmd.AddCommand(exportCmd) diff --git a/modrinth/export.go b/modrinth/export.go index 009589f..130bbf7 100644 --- a/modrinth/export.go +++ b/modrinth/export.go @@ -5,14 +5,12 @@ import ( "encoding/json" "fmt" "github.com/spf13/viper" - "net/url" "os" "path/filepath" "strconv" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" - "golang.org/x/exp/slices" ) // exportCmd represents the export command @@ -57,7 +55,12 @@ var exportCmd = &cobra.Command{ // TODO: should index just expose indexPath itself, through a function? indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File)) - mods, unwhitelistedMods := loadMods(index) + fmt.Println("Reading external files...") + mods, err := index.LoadAllMods() + if err != nil { + fmt.Printf("Error reading file: %v\n", err) + os.Exit(1) + } fileName := viper.GetString("modrinth.export.output") if fileName == "" { @@ -77,8 +80,11 @@ var exportCmd = &cobra.Command{ os.Exit(1) } - // TODO: cache these (ideally with changes to pack format) - fmt.Println("Retrieving hashes for external mods...") + // TODO: finish updating to use download session + fmt.Println("Retrieving external mods...") + session, err := core.CreateDownloadSession(mods, []string{"sha1", "sha512", "length-bytes"}) + _ = session + modsHashes := make([]map[string]string, len(mods)) for i, mod := range mods { modsHashes[i], err = mod.GetHashes([]string{"sha1", "sha512", "length-bytes"}) @@ -218,6 +224,8 @@ var exportCmd = &cobra.Command{ } } + // TODO: get rid of this, do whitelist checks elsewhere + if len(unwhitelistedMods) > 0 { fmt.Println("Downloading unwhitelisted mods...") } @@ -267,6 +275,7 @@ var exportCmd = &cobra.Command{ }, } +// TODO: update whitelist var whitelistedHosts = []string{ "cdn.modrinth.com", "edge.forgecdn.net", @@ -274,33 +283,14 @@ var whitelistedHosts = []string{ "raw.githubusercontent.com", } -func loadMods(index core.Index) ([]core.Mod, []core.Mod) { - modPaths := index.GetAllMods() - mods := make([]core.Mod, 0, len(modPaths)) - unwhitelistedMods := make([]core.Mod, 0) - fmt.Println("Reading mod files...") - for _, v := range modPaths { - modData, err := core.LoadMod(v) - if err != nil { - fmt.Printf("Error reading mod file %s: %s\n", v, err.Error()) - // TODO: exit(1)? - continue - } - - modUrl, err := url.Parse(modData.Download.URL) - if err == nil { - if slices.Contains(whitelistedHosts, modUrl.Host) { - mods = append(mods, modData) - } else { - unwhitelistedMods = append(unwhitelistedMods, modData) - } - } else { - fmt.Printf("Failed to parse mod URL: %v\n", modUrl) - mods = append(mods, modData) - } - } - return mods, unwhitelistedMods -} +//modUrl, err := url.Parse(modData.Download.URL) +//if err == nil { +//if slices.Contains(whitelistedHosts, modUrl.Host) { +//mods = append(mods, modData) +//} else { +//unwhitelistedMods = append(unwhitelistedMods, modData) +//} +//} func init() { modrinthCmd.AddCommand(exportCmd)