Support Modrinth modpack fileSize field and sha512 hash

core/hash.go

@@ -15,34 +15,76 @@ import (
 )
 
 // GetHashImpl gets an implementation of hash.Hash for the given hash type string
-func GetHashImpl(hashType string) (hash.Hash, HashStringer, error) {
+func GetHashImpl(hashType string) (HashStringer, error) {
 	switch strings.ToLower(hashType) {
 	case "sha1":
-		return sha1.New(), hexStringer{}, nil
+		return hexStringer{sha1.New()}, nil
 	case "sha256":
-		return sha256.New(), hexStringer{}, nil
+		return hexStringer{sha256.New()}, nil
 	case "sha512":
-		return sha512.New(), hexStringer{}, nil
+		return hexStringer{sha512.New()}, nil
 	case "md5":
-		return md5.New(), hexStringer{}, nil
+		return hexStringer{md5.New()}, nil
-	case "murmur2":
+	case "murmur2": // TODO: change to something indicating that this is the CF variant
-		return murmur2.New(), numberStringer{}, nil
+		return number32As64Stringer{murmur2.New()}, nil
+	case "length-bytes": // TODO: only used internally for now; should not be saved
+		return number64Stringer{&LengthHasher{}}, nil
 	}
-	return nil, nil, fmt.Errorf("hash implementation %s not found", hashType)
+	return nil, fmt.Errorf("hash implementation %s not found", hashType)
 }
 
 type HashStringer interface {
+	hash.Hash
 	HashToString([]byte) string
 }
 
-type hexStringer struct{}
+type hexStringer struct {
+	hash.Hash
+}
 
 func (hexStringer) HashToString(data []byte) string {
 	return hex.EncodeToString(data)
 }
 
-type numberStringer struct{}
+type number32As64Stringer struct {
+	hash.Hash
+}
 
-func (numberStringer) HashToString(data []byte) string {
+func (number32As64Stringer) HashToString(data []byte) string {
 	return strconv.FormatUint(uint64(binary.BigEndian.Uint32(data)), 10)
 }
+
+type number64Stringer struct {
+	hash.Hash
+}
+
+func (number64Stringer) HashToString(data []byte) string {
+	return strconv.FormatUint(binary.BigEndian.Uint64(data), 10)
+}
+
+type LengthHasher struct {
+	length uint64
+}
+
+func (h *LengthHasher) Write(p []byte) (n int, err error) {
+	h.length += uint64(len(p))
+	return len(p), nil
+}
+
+func (h *LengthHasher) Sum(b []byte) []byte {
+	ext := append(b, make([]byte, 8)...)
+	binary.BigEndian.PutUint64(ext, h.length)
+	return ext
+}
+
+func (h *LengthHasher) Size() int {
+	return 8
+}
+
+func (h *LengthHasher) BlockSize() int {
+	return 1
+}
+
+func (h *LengthHasher) Reset() {
+	h.length = 0
+}

core/index.go

@@ -133,7 +133,7 @@ func (in *Index) updateFile(path string) error {
 		// Hash usage strategy (may change):
 		// Just use SHA256, overwrite existing hash regardless of what it is
 		// May update later to continue using the same hash that was already being used
-		h, stringer, err := GetHashImpl("sha256")
+		h, err := GetHashImpl("sha256")
 		if err != nil {
 			_ = f.Close()
 			return err
@@ -146,7 +146,7 @@ func (in *Index) updateFile(path string) error {
 		if err != nil {
 			return err
 		}
-		hashString = stringer.HashToString(h.Sum(nil))
+		hashString = h.HashToString(h.Sum(nil))
 	}
 
 	mod := false
@@ -377,7 +377,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error {
 	if err != nil {
 		return err
 	}
-	h, stringer, err := GetHashImpl(hashFormat)
+	h, err := GetHashImpl(hashFormat)
 	if err != nil {
 		return err
 	}
@@ -388,7 +388,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error {
 		return err
 	}
 
-	calculatedHash := stringer.HashToString(h.Sum(nil))
+	calculatedHash := h.HashToString(h.Sum(nil))
 	if calculatedHash != f.Hash && !viper.GetBool("no-internal-hashes") {
 		return errors.New("hash of saved file is invalid")
 	}

core/mod.go

@@ -3,6 +3,7 @@ package core
 import (
 	"errors"
 	"fmt"
+	"golang.org/x/exp/slices"
 	"io"
 	"net/http"
 	"os"
@@ -92,7 +93,7 @@ func (m Mod) Write() (string, string, error) {
 		}
 	}
 
-	h, stringer, err := GetHashImpl("sha256")
+	h, err := GetHashImpl("sha256")
 	if err != nil {
 		_ = f.Close()
 		return "", "", err
@@ -103,7 +104,7 @@ func (m Mod) Write() (string, string, error) {
 	// Disable indentation
 	enc.Indent = ""
 	err = enc.Encode(m)
-	hashString := stringer.HashToString(h.Sum(nil))
+	hashString := h.HashToString(h.Sum(nil))
 	if err != nil {
 		_ = f.Close()
 		return "sha256", hashString, err
@@ -130,6 +131,7 @@ func (m Mod) GetDestFilePath() string {
 // DownloadFile attempts to resolve and download the file
 func (m Mod) DownloadFile(dest io.Writer) error {
 	resp, err := http.Get(m.Download.URL)
+	// TODO: content type, user-agent?
 	if err != nil {
 		return err
 	}
@@ -137,9 +139,9 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 		_ = resp.Body.Close()
 		return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
 	}
-	h, stringer, err := GetHashImpl(m.Download.HashFormat)
+	h, err := GetHashImpl(m.Download.HashFormat)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
 	}
 
 	w := io.MultiWriter(h, dest)
@@ -148,7 +150,7 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 		return err
 	}
 
-	calculatedHash := stringer.HashToString(h.Sum(nil))
+	calculatedHash := h.HashToString(h.Sum(nil))
 
 	// Check if the hash of the downloaded file matches the expected hash.
 	if calculatedHash != m.Download.Hash {
@@ -157,3 +159,72 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 
 	return nil
 }
+
+// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary
+func (m Mod) GetHashes(hashes []string) (map[string]string, error) {
+	out := make(map[string]string)
+
+	// Get the hash already stored TODO: store multiple (requires breaking pack change)
+	if m.Download.Hash != "" {
+		idx := slices.Index(hashes, m.Download.HashFormat)
+		if idx > -1 {
+			out[m.Download.HashFormat] = m.Download.Hash
+			// Remove hash from list to retrieve
+			hashes = slices.Delete(hashes, idx, idx+1)
+		}
+	}
+
+	// Retrieve the remaining hashes
+	if len(hashes) > 0 {
+		resp, err := http.Get(m.Download.URL)
+		// TODO: content type, user-agent?
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode != 200 {
+			_ = resp.Body.Close()
+			return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
+		}
+
+		// Special fast-path for file length only
+		if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 {
+			out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10)
+			_ = resp.Body.Close()
+			return out, nil
+		}
+
+		mainHasher, err := GetHashImpl(m.Download.HashFormat)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
+		}
+
+		hashers := make([]HashStringer, len(hashes))
+		allHashers := make([]io.Writer, len(hashers))
+		for i, v := range hashes {
+			hashers[i], err = GetHashImpl(v)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err)
+			}
+			allHashers[i] = hashers[i]
+		}
+		allHashers = append(allHashers, mainHasher)
+
+		w := io.MultiWriter(allHashers...)
+		_, err = io.Copy(w, resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download file: %w", err)
+		}
+
+		calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil))
+
+		// Check if the hash of the downloaded file matches the expected hash
+		if calculatedHash != m.Download.Hash {
+			return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
+		}
+
+		for i, v := range hashers {
+			out[hashes[i]] = v.HashToString(v.Sum(nil))
+		}
+	}
+	return out, nil
+}

core/pack.go

@@ -116,7 +116,7 @@ func (pack *Pack) UpdateIndexHash() error {
 	// Hash usage strategy (may change):
 	// Just use SHA256, overwrite existing hash regardless of what it is
 	// May update later to continue using the same hash that was already being used
-	h, stringer, err := GetHashImpl("sha256")
+	h, err := GetHashImpl("sha256")
 	if err != nil {
 		_ = f.Close()
 		return err
@@ -125,7 +125,7 @@ func (pack *Pack) UpdateIndexHash() error {
 		_ = f.Close()
 		return err
 	}
-	hashString := stringer.HashToString(h.Sum(nil))
+	hashString := h.HashToString(h.Sum(nil))
 
 	pack.Index.HashFormat = "sha256"
 	pack.Index.Hash = hashString

modrinth/export.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/packwiz/packwiz/core"
 	"github.com/spf13/cobra"
@@ -77,26 +78,15 @@ var exportCmd = &cobra.Command{
 		}
 
 		// TODO: cache these (ideally with changes to pack format)
-		fmt.Println("Retrieving SHA1 hashes for external mods...")
+		fmt.Println("Retrieving hashes for external mods...")
-		sha1Hashes := make([]string, len(mods))
+		modsHashes := make([]map[string]string, len(mods))
 		for i, mod := range mods {
-			if mod.Download.HashFormat == "sha1" {
+			modsHashes[i], err = mod.GetHashes([]string{"sha1", "sha512", "length-bytes"})
-				sha1Hashes[i] = mod.Download.Hash
+			if err != nil {
-			} else {
+				fmt.Printf("Error downloading mod file %s: %s\n", mod.Download.URL, err.Error())
-				// Hash format for this mod isn't SHA1 - and the Modrinth pack format requires it; so get it by downloading the file
+				continue
-				h, stringer, err := core.GetHashImpl("sha1")
-				if err != nil {
-					panic("Failed to get sha1 hash implementation")
-				}
-				err = mod.DownloadFile(h)
-				if err != nil {
-					fmt.Printf("Error downloading mod file %s: %s\n", mod.Download.URL, err.Error())
-					// TODO: exit(1)?
-					continue
-				}
-				sha1Hashes[i] = stringer.HashToString(h.Sum(nil))
-				fmt.Printf("Retrieved SHA1 hash for %s successfully\n", mod.Download.URL)
 			}
+			fmt.Printf("Retrieved hashes for %s successfully\n", mod.Download.URL)
 		}
 
 		manifestFile, err := exp.Create("modrinth.index.json")
@@ -119,7 +109,12 @@ var exportCmd = &cobra.Command{
 			path := filepath.ToSlash(pathForward)
 
 			hashes := make(map[string]string)
-			hashes["sha1"] = sha1Hashes[i]
+			hashes["sha1"] = modsHashes[i]["sha1"]
+			hashes["sha512"] = modsHashes[i]["sha512"]
+			fileSize, err := strconv.ParseUint(modsHashes[i]["length-bytes"], 10, 64)
+			if err != nil {
+				panic(err)
+			}
 
 			// Create env options based on configured optional/side
 			var envInstalled string
@@ -155,6 +150,7 @@ var exportCmd = &cobra.Command{
 					Server string `json:"server"`
 				}{Client: clientEnv, Server: serverEnv},
 				Downloads: []string{u},
+				FileSize:  uint32(fileSize),
 			}
 		}
 

modrinth/pack.go

@@ -18,4 +18,5 @@ type PackFile struct {
 		Server string `json:"server"`
 	} `json:"env"`
 	Downloads []string `json:"downloads"`
+	FileSize  uint32   `json:"fileSize"`
 }