mirrors/packwiz
1
0
Fork 0
mirror of https://github.com/packwiz/packwiz.git synced 2025-04-19 13:06:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

WIP: misc fixes, start updating CF/MR export to use download system

Browse Source
This commit is contained in:
comp500 2022-05-20 18:13:43 +01:00
parent e73fa8c48a
commit 55f0e4a297
8 changed files with 182 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cmdshared/downloadutil.go Normal file
View File

@ -0,0 +1,32 @@
package cmdshared
import (
"fmt"
"github.com/packwiz/packwiz/core"
"os"
"path/filepath"
)
func ListManualDownloads(session core.DownloadSession) {
manualDownloads := session.GetManualDownloads()
if len(manualDownloads) > 0 {
fmt.Printf("Found %v manual downloads; these mods are unable to be downloaded by packwiz (due to API limitations) and must be manually downloaded:\n",
len(manualDownloads))
for _, dl := range manualDownloads {
fmt.Printf("%s (%s) from %s\n", dl.Name, dl.FileName, dl.URL)
}
cacheDir, err := core.GetPackwizCache()
if err != nil {
fmt.Printf("Error locating cache folder: %v", err)
os.Exit(1)
}
err = os.MkdirAll(filepath.Join(cacheDir, core.DownloadCacheInFolder), 0755)
if err != nil {
fmt.Printf("Error creating cache in folder: %v", err)
os.Exit(1)
}
fmt.Printf("Once you have done so, place these files in %s and re-run this command.\n",
filepath.Join(cacheDir, core.DownloadCacheInFolder))
os.Exit(1)
}
}

84
core/download.go
View File

@ -13,6 +13,8 @@ import (
"strings"
)
const DownloadCacheInFolder = "in"
type DownloadSession interface {
GetManualDownloads() []ManualDownload
StartDownloads() chan CompletedDownload
@ -20,9 +22,9 @@ type DownloadSession interface {
}
type CompletedDownload struct {
File *os.File
DestFilePath string
Hashes map[string]string
File *os.File
Mod *Mod
Hashes map[string]string
// Error indicates if/why downloading this file failed
Error error
// Warnings indicates messages to show to the user regarding this file (download was successful, but had a problem)
@ -39,7 +41,7 @@ type downloadSessionInternal struct {
type downloadTask struct {
metaDownloaderData MetaDownloaderData
destFilePath string
mod *Mod
url string
hashFormat string
hash string
@ -52,11 +54,23 @@ func (d *downloadSessionInternal) GetManualDownloads() []ManualDownload {
func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload {
downloads := make(chan CompletedDownload)
for _, task := range d.downloadTasks {
// Get handle for mod
cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash)
if cacheHandle != nil {
download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.destFilePath)
go func() {
for _, task := range d.downloadTasks {
// Get handle for mod
cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash)
if cacheHandle != nil {
download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.mod)
if err != nil {
downloads <- CompletedDownload{
Error: err,
}
} else {
downloads <- download
}
continue
}
download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex)
if err != nil {
downloads <- CompletedDownload{
Error: err,
@ -64,18 +78,9 @@ func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload {
} else {
downloads <- download
}
continue
}
download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex)
if err != nil {
downloads <- CompletedDownload{
Error: err,
}
} else {
downloads <- download
}
}
close(downloads)
}()
return downloads
}
@ -91,7 +96,7 @@ func (d *downloadSessionInternal) SaveIndex() error {
return nil
}
func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, destFilePath string) (CompletedDownload, error) {
func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, mod *Mod) (CompletedDownload, error) {
// Already stored; try using it!
file, err := cacheHandle.Open()
if err == nil {
@ -111,9 +116,9 @@ func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, d
}
return CompletedDownload{
File: file,
DestFilePath: destFilePath,
Hashes: cacheHandle.Hashes,
File: file,
Mod: mod,
Hashes: cacheHandle.Hashes,
}, nil
} else {
return CompletedDownload{}, fmt.Errorf("failed to read file %s from cache: %w", cacheHandle.Path(), err)
@ -151,7 +156,7 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st
err = teeHashes(hashesToObtain, hashes, tempFile, data)
_ = data.Close()
if err != nil {
return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.destFilePath, err)
return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.mod.Name, err)
}
}
@ -180,14 +185,14 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st
}
return CompletedDownload{
File: file,
DestFilePath: task.destFilePath,
Hashes: hashes,
Warnings: warnings,
File: file,
Mod: task.mod,
Hashes: hashes,
Warnings: warnings,
}, nil
}
func selectPreferredHash(hashes map[string]string) (currHash string, currHashFormat string) {
func selectPreferredHash(hashes map[string]string) (currHashFormat string, currHash string) {
for _, hashFormat := range preferredHashList {
if hash, ok := hashes[hashFormat]; ok {
currHashFormat = hashFormat
@ -226,7 +231,7 @@ func teeHashes(hashesToObtain []string, hashes map[string]string,
return fmt.Errorf("failed to get hash format %s", validateHashFormat)
}
hashers := make(map[string]HashStringer, len(hashesToObtain))
allWriters := make([]io.Writer, len(hashers))
allWriters := make([]io.Writer, len(hashesToObtain))
for i, v := range hashesToObtain {
hashers[v], err = GetHashImpl(v)
if err != nil {
@ -340,11 +345,11 @@ func (h *CacheIndexHandle) CreateFromTemp(temp *os.File) (*os.File, error) {
if err != nil {
return nil, err
}
err = os.Rename(temp.Name(), h.Path())
err = temp.Close()
if err != nil {
return nil, err
}
err = temp.Close()
err = os.Rename(temp.Name(), h.Path())
if err != nil {
return nil, err
}
@ -425,12 +430,12 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio
// Get necessary metadata for all files
for _, mod := range mods {
if mod.Download.Mode == "url" {
if mod.Download.Mode == "url" || mod.Download.Mode == "" {
downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{
destFilePath: mod.GetDestFilePath(),
url: mod.Download.URL,
hashFormat: mod.Download.HashFormat,
hash: mod.Download.Hash,
mod: mod,
url: mod.Download.URL,
hashFormat: mod.Download.HashFormat,
hash: mod.Download.Hash,
})
} else if strings.HasPrefix(mod.Download.Mode, "metadata:") {
dlID := strings.TrimPrefix(mod.Download.Mode, "metadata:")
@ -452,10 +457,11 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio
for i, v := range mods {
isManual, manualDownload := meta[i].GetManualDownload()
if isManual {
// TODO: lookup in index!
downloadSession.manualDownloads = append(downloadSession.manualDownloads, manualDownload)
} else {
downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{
destFilePath: v.GetDestFilePath(),
mod: v,
metaDownloaderData: meta[i],
hashFormat: v.Download.HashFormat,
hash: v.Download.Hash,

15
core/index.go
View File

@ -2,6 +2,7 @@ package core
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
@ -355,6 +356,20 @@ func (in Index) GetAllMods() []string {
return list
}
// LoadAllMods reads all metadata files into Mod structs
func (in Index) LoadAllMods() ([]*Mod, error) {
modPaths := in.GetAllMods()
mods := make([]*Mod, len(modPaths))
for i, v := range modPaths {
modData, err := LoadMod(v)
if err != nil {
return nil, fmt.Errorf("failed to read metadata file %s: %w", v, err)
}
mods[i] = &modData
}
return mods, nil
}
// GetFilePath attempts to get the path of the destination index file as it is stored on disk
func (in Index) GetFilePath(f IndexFile) string {
return filepath.Join(filepath.Dir(in.indexFile), filepath.FromSlash(f.File))

1
core/interfaces.go
View File

@ -51,6 +51,5 @@ type MetaDownloaderData interface {
type ManualDownload struct {
Name string
FileName string
DestPath string
URL string
}

110
core/mod.go
View File

@ -2,15 +2,10 @@ package core
import (
"errors"
"fmt"
"golang.org/x/exp/slices"
"github.com/BurntSushi/toml"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"github.com/BurntSushi/toml"
)
// Mod stores metadata about a mod. This is written to a TOML file for each mod.
@ -134,106 +129,3 @@ func (m Mod) GetFilePath() string {
func (m Mod) GetDestFilePath() string {
return filepath.Join(filepath.Dir(m.metaFile), filepath.FromSlash(m.FileName))
}
// DownloadFile attempts to resolve and download the file
func (m Mod) DownloadFile(dest io.Writer) error {
// TODO: check mode
resp, err := http.Get(m.Download.URL)
// TODO: content type, user-agent?
if err != nil {
return err
}
if resp.StatusCode != 200 {
_ = resp.Body.Close()
return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
}
h, err := GetHashImpl(m.Download.HashFormat)
if err != nil {
return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
}
w := io.MultiWriter(h, dest)
_, err = io.Copy(w, resp.Body)
if err != nil {
return err
}
calculatedHash := h.HashToString(h.Sum(nil))
// Check if the hash of the downloaded file matches the expected hash.
if calculatedHash != m.Download.Hash {
return fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
}
return nil
}
// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary
func (m Mod) GetHashes(hashes []string) (map[string]string, error) {
out := make(map[string]string)
// Get the hash already stored TODO: store multiple (requires breaking pack change)
if m.Download.Hash != "" {
idx := slices.Index(hashes, m.Download.HashFormat)
if idx > -1 {
out[m.Download.HashFormat] = m.Download.Hash
// Remove hash from list to retrieve
hashes = slices.Delete(hashes, idx, idx+1)
}
}
// Retrieve the remaining hashes
if len(hashes) > 0 {
// TODO: check mode
resp, err := http.Get(m.Download.URL)
// TODO: content type, user-agent?
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
_ = resp.Body.Close()
return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
}
// Special fast-path for file length only
if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 {
out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10)
_ = resp.Body.Close()
return out, nil
}
mainHasher, err := GetHashImpl(m.Download.HashFormat)
if err != nil {
return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
}
hashers := make([]HashStringer, len(hashes))
allHashers := make([]io.Writer, len(hashers))
for i, v := range hashes {
hashers[i], err = GetHashImpl(v)
if err != nil {
return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err)
}
allHashers[i] = hashers[i]
}
allHashers = append(allHashers, mainHasher)
w := io.MultiWriter(allHashers...)
_, err = io.Copy(w, resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to download file: %w", err)
}
calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil))
// Check if the hash of the downloaded file matches the expected hash
if calculatedHash != m.Download.Hash {
return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
}
for i, v := range hashers {
out[hashes[i]] = v.HashToString(v.Sum(nil))
}
}
return out, nil
}

9
curseforge/curseforge.go
View File

@ -194,6 +194,7 @@ func createModFile(modInfo modInfo, fileInfo modFileInfo, index *core.Index, opt
Download: core.ModDownload{
HashFormat: hashFormat,
Hash: hash,
Mode: "metadata:curseforge",
},
Option: optional,
Update: updateMap,
@ -422,6 +423,7 @@ func (u cfUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error {
v.Download = core.ModDownload{
HashFormat: hashFormat,
Hash: hash,
Mode: "metadata:curseforge",
}
v.Update["curseforge"]["project-id"] = modState.ID
@ -481,9 +483,10 @@ func (c cfDownloader) GetFilesMetadata(mods []*core.Mod) ([]core.MetaDownloaderD
meta.websiteUrl = meta.websiteUrl + "/files/" + strconv.Itoa(fileInfo.ID)
meta.fileName = fileInfo.FileName
}
}
downloaderData[indexMap[modID]] = &cfDownloadMetadata{
url: fileInfo.DownloadURL,
} else {
downloaderData[indexMap[modID]] = &cfDownloadMetadata{
url: fileInfo.DownloadURL,
}
}
}

100
curseforge/export.go
View File

@ -4,14 +4,15 @@ import (
"archive/zip"
"bufio"
"fmt"
"github.com/packwiz/packwiz/cmdshared"
"github.com/packwiz/packwiz/core"
"github.com/packwiz/packwiz/curseforge/packinterop"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"io"
"os"
"path/filepath"
"strconv"
"github.com/packwiz/packwiz/core"
"github.com/spf13/cobra"
)
// exportCmd represents the export command
@ -41,28 +42,33 @@ var exportCmd = &cobra.Command{
err = index.Refresh()
if err != nil {
fmt.Println(err)
return
os.Exit(1)
}
err = index.Write()
if err != nil {
fmt.Println(err)
return
os.Exit(1)
}
err = pack.UpdateIndexHash()
if err != nil {
fmt.Println(err)
return
os.Exit(1)
}
err = pack.Write()
if err != nil {
fmt.Println(err)
return
os.Exit(1)
}
// TODO: should index just expose indexPath itself, through a function?
indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File))
mods := loadMods(index)
fmt.Println("Reading external files...")
mods, err := index.LoadAllMods()
if err != nil {
fmt.Printf("Error reading file: %v\n", err)
os.Exit(1)
}
i := 0
// Filter mods by side
// TODO: opt-in optional disabled filtering?
@ -104,11 +110,13 @@ var exportCmd = &cobra.Command{
}
cfFileRefs := make([]packinterop.AddonFileReference, 0, len(mods))
nonCfMods := make([]*core.Mod, 0)
for _, mod := range mods {
projectRaw, ok := mod.GetParsedUpdateData("curseforge")
// If the mod has curseforge metadata, add it to cfFileRefs
// TODO: how to handle files with CF metadata, but with different download path?
if ok {
// TODO: change back to use ok
_ = ok
if false {
p := projectRaw.(cfUpdateData)
cfFileRefs = append(cfFileRefs, packinterop.AddonFileReference{
ProjectID: p.ProjectID,
@ -116,26 +124,59 @@ var exportCmd = &cobra.Command{
OptionalDisabled: mod.Option != nil && mod.Option.Optional && !mod.Option.Default,
})
} else {
// If the mod doesn't have the metadata, save it into the zip
path, err := filepath.Rel(filepath.Dir(indexPath), mod.GetDestFilePath())
nonCfMods = append(nonCfMods, mod)
}
}
// Download external files and save directly into the zip
if len(nonCfMods) > 0 {
fmt.Printf("Retrieving %v external files to store in the modpack zip...\n", len(nonCfMods))
fmt.Println("Disclaimer: you are responsible for ensuring you comply with ALL the licenses, or obtain appropriate permissions, for the files listed below")
fmt.Println("Note that mods bundled within a CurseForge pack must be in the Approved Non-CurseForge Mods list")
fmt.Println("packwiz is currently unable to match metadata between mod sites - if any of these are available from CurseForge you should change them to use CurseForge metadata (e.g. by reinstalling them using the cf commands)")
fmt.Println()
session, err := core.CreateDownloadSession(nonCfMods, []string{})
if err != nil {
fmt.Printf("Error retrieving external files: %v\n", err)
os.Exit(1)
}
cmdshared.ListManualDownloads(session)
for dl := range session.StartDownloads() {
if dl.Error != nil {
// TODO: ensure name is populated
fmt.Printf("Download of %s (%s) failed: %v\n", dl.Mod.Name, dl.Mod.FileName, dl.Error)
continue
}
for warning := range dl.Warnings {
// TODO: get name
fmt.Printf("Download warning: %v\n", warning)
}
path, err := filepath.Rel(filepath.Dir(indexPath), dl.Mod.GetDestFilePath())
if err != nil {
fmt.Printf("Error resolving mod file: %s\n", err.Error())
// TODO: exit(1)?
fmt.Printf("Error resolving mod file: %v\n", err)
continue
}
modFile, err := exp.Create(filepath.ToSlash(filepath.Join("overrides", path)))
if err != nil {
fmt.Printf("Error creating mod file %s: %s\n", path, err.Error())
// TODO: exit(1)?
fmt.Printf("Error creating mod file %s: %v\n", path, err)
continue
}
err = mod.DownloadFile(modFile)
_, err = io.Copy(modFile, dl.File)
if err != nil {
fmt.Printf("Error downloading mod file %s: %s\n", path, err.Error())
// TODO: exit(1)?
fmt.Printf("Error copying file %s: %v\n", path, err)
continue
}
}
err = session.SaveIndex()
if err != nil {
fmt.Printf("Error saving cache index: %v\n", err)
os.Exit(1)
}
}
manifestFile, err := exp.Create("manifest.json")
@ -203,7 +244,7 @@ var exportCmd = &cobra.Command{
},
}
func createModlist(zw *zip.Writer, mods []core.Mod) error {
func createModlist(zw *zip.Writer, mods []*core.Mod) error {
modlistFile, err := zw.Create("modlist.html")
if err != nil {
return err
@ -239,25 +280,6 @@ func createModlist(zw *zip.Writer, mods []core.Mod) error {
return w.Flush()
}
func loadMods(index core.Index) []core.Mod {
modPaths := index.GetAllMods()
mods := make([]core.Mod, len(modPaths))
i := 0
fmt.Println("Reading mod files...")
for _, v := range modPaths {
modData, err := core.LoadMod(v)
if err != nil {
fmt.Printf("Error reading mod file %s: %s\n", v, err.Error())
// TODO: exit(1)?
continue
}
mods[i] = modData
i++
}
return mods[:i]
}
func init() {
curseforgeCmd.AddCommand(exportCmd)

54
modrinth/export.go
View File

@ -5,14 +5,12 @@ import (
"encoding/json"
"fmt"
"github.com/spf13/viper"
"net/url"
"os"
"path/filepath"
"strconv"
"github.com/packwiz/packwiz/core"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
)
// exportCmd represents the export command
@ -57,7 +55,12 @@ var exportCmd = &cobra.Command{
// TODO: should index just expose indexPath itself, through a function?
indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File))
mods, unwhitelistedMods := loadMods(index)
fmt.Println("Reading external files...")
mods, err := index.LoadAllMods()
if err != nil {
fmt.Printf("Error reading file: %v\n", err)
os.Exit(1)
}
fileName := viper.GetString("modrinth.export.output")
if fileName == "" {
@ -77,8 +80,11 @@ var exportCmd = &cobra.Command{
os.Exit(1)
}
// TODO: cache these (ideally with changes to pack format)
fmt.Println("Retrieving hashes for external mods...")
// TODO: finish updating to use download session
fmt.Println("Retrieving external mods...")
session, err := core.CreateDownloadSession(mods, []string{"sha1", "sha512", "length-bytes"})
_ = session
modsHashes := make([]map[string]string, len(mods))
for i, mod := range mods {
modsHashes[i], err = mod.GetHashes([]string{"sha1", "sha512", "length-bytes"})
@ -218,6 +224,8 @@ var exportCmd = &cobra.Command{
}
}
// TODO: get rid of this, do whitelist checks elsewhere
if len(unwhitelistedMods) > 0 {
fmt.Println("Downloading unwhitelisted mods...")
}
@ -267,6 +275,7 @@ var exportCmd = &cobra.Command{
},
}
// TODO: update whitelist
var whitelistedHosts = []string{
"cdn.modrinth.com",
"edge.forgecdn.net",
@ -274,33 +283,14 @@ var whitelistedHosts = []string{
"raw.githubusercontent.com",
}
func loadMods(index core.Index) ([]core.Mod, []core.Mod) {
modPaths := index.GetAllMods()
mods := make([]core.Mod, 0, len(modPaths))
unwhitelistedMods := make([]core.Mod, 0)
fmt.Println("Reading mod files...")
for _, v := range modPaths {
modData, err := core.LoadMod(v)
if err != nil {
fmt.Printf("Error reading mod file %s: %s\n", v, err.Error())
// TODO: exit(1)?
continue
}
modUrl, err := url.Parse(modData.Download.URL)
if err == nil {
if slices.Contains(whitelistedHosts, modUrl.Host) {
mods = append(mods, modData)
} else {
unwhitelistedMods = append(unwhitelistedMods, modData)
}
} else {
fmt.Printf("Failed to parse mod URL: %v\n", modUrl)
mods = append(mods, modData)
}
}
return mods, unwhitelistedMods
}
//modUrl, err := url.Parse(modData.Download.URL)
//if err == nil {
//if slices.Contains(whitelistedHosts, modUrl.Host) {
//mods = append(mods, modData)
//} else {
//unwhitelistedMods = append(unwhitelistedMods, modData)
//}
//}
func init() {
modrinthCmd.AddCommand(exportCmd)