Merge branch 'master' into wip/new-curseforge-api

core/hash.go

@@ -15,34 +15,76 @@ import (
 )
 
 // GetHashImpl gets an implementation of hash.Hash for the given hash type string
-func GetHashImpl(hashType string) (hash.Hash, HashStringer, error) {
+func GetHashImpl(hashType string) (HashStringer, error) {
 	switch strings.ToLower(hashType) {
 	case "sha1":
-		return sha1.New(), hexStringer{}, nil
+		return hexStringer{sha1.New()}, nil
 	case "sha256":
-		return sha256.New(), hexStringer{}, nil
+		return hexStringer{sha256.New()}, nil
 	case "sha512":
-		return sha512.New(), hexStringer{}, nil
+		return hexStringer{sha512.New()}, nil
 	case "md5":
-		return md5.New(), hexStringer{}, nil
-	case "murmur2":
-		return murmur2.New(), numberStringer{}, nil
+		return hexStringer{md5.New()}, nil
+	case "murmur2": // TODO: change to something indicating that this is the CF variant
+		return number32As64Stringer{murmur2.New()}, nil
+	case "length-bytes": // TODO: only used internally for now; should not be saved
+		return number64Stringer{&LengthHasher{}}, nil
 	}
-	return nil, nil, fmt.Errorf("hash implementation %s not found", hashType)
+	return nil, fmt.Errorf("hash implementation %s not found", hashType)
 }
 
 type HashStringer interface {
+	hash.Hash
 	HashToString([]byte) string
 }
 
-type hexStringer struct{}
+type hexStringer struct {
+	hash.Hash
+}
 
 func (hexStringer) HashToString(data []byte) string {
 	return hex.EncodeToString(data)
 }
 
-type numberStringer struct{}
+type number32As64Stringer struct {
+	hash.Hash
+}
 
-func (numberStringer) HashToString(data []byte) string {
+func (number32As64Stringer) HashToString(data []byte) string {
 	return strconv.FormatUint(uint64(binary.BigEndian.Uint32(data)), 10)
 }
+
+type number64Stringer struct {
+	hash.Hash
+}
+
+func (number64Stringer) HashToString(data []byte) string {
+	return strconv.FormatUint(binary.BigEndian.Uint64(data), 10)
+}
+
+type LengthHasher struct {
+	length uint64
+}
+
+func (h *LengthHasher) Write(p []byte) (n int, err error) {
+	h.length += uint64(len(p))
+	return len(p), nil
+}
+
+func (h *LengthHasher) Sum(b []byte) []byte {
+	ext := append(b, make([]byte, 8)...)
+	binary.BigEndian.PutUint64(ext, h.length)
+	return ext
+}
+
+func (h *LengthHasher) Size() int {
+	return 8
+}
+
+func (h *LengthHasher) BlockSize() int {
+	return 1
+}
+
+func (h *LengthHasher) Reset() {
+	h.length = 0
+}

core/index.go

@@ -133,7 +133,7 @@ func (in *Index) updateFile(path string) error {
 		// Hash usage strategy (may change):
 		// Just use SHA256, overwrite existing hash regardless of what it is
 		// May update later to continue using the same hash that was already being used
-		h, stringer, err := GetHashImpl("sha256")
+		h, err := GetHashImpl("sha256")
 		if err != nil {
 			_ = f.Close()
 			return err
@@ -146,7 +146,7 @@ func (in *Index) updateFile(path string) error {
 		if err != nil {
 			return err
 		}
-		hashString = stringer.HashToString(h.Sum(nil))
+		hashString = h.HashToString(h.Sum(nil))
 	}
 
 	mod := false
@@ -370,7 +370,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error {
 	if err != nil {
 		return err
 	}
-	h, stringer, err := GetHashImpl(hashFormat)
+	h, err := GetHashImpl(hashFormat)
 	if err != nil {
 		return err
 	}
@@ -381,7 +381,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error {
 		return err
 	}
 
-	calculatedHash := stringer.HashToString(h.Sum(nil))
+	calculatedHash := h.HashToString(h.Sum(nil))
 	if calculatedHash != f.Hash && !viper.GetBool("no-internal-hashes") {
 		return errors.New("hash of saved file is invalid")
 	}

core/mod.go

@@ -3,6 +3,7 @@ package core
 import (
 	"errors"
 	"fmt"
+	"golang.org/x/exp/slices"
 	"io"
 	"net/http"
 	"os"
@@ -92,7 +93,7 @@ func (m Mod) Write() (string, string, error) {
 		}
 	}
 
-	h, stringer, err := GetHashImpl("sha256")
+	h, err := GetHashImpl("sha256")
 	if err != nil {
 		_ = f.Close()
 		return "", "", err
@@ -103,7 +104,7 @@ func (m Mod) Write() (string, string, error) {
 	// Disable indentation
 	enc.Indent = ""
 	err = enc.Encode(m)
-	hashString := stringer.HashToString(h.Sum(nil))
+	hashString := h.HashToString(h.Sum(nil))
 	if err != nil {
 		_ = f.Close()
 		return "sha256", hashString, err
@@ -130,6 +131,7 @@ func (m Mod) GetDestFilePath() string {
 // DownloadFile attempts to resolve and download the file
 func (m Mod) DownloadFile(dest io.Writer) error {
 	resp, err := http.Get(m.Download.URL)
+	// TODO: content type, user-agent?
 	if err != nil {
 		return err
 	}
@@ -137,9 +139,9 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 		_ = resp.Body.Close()
 		return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
 	}
-	h, stringer, err := GetHashImpl(m.Download.HashFormat)
+	h, err := GetHashImpl(m.Download.HashFormat)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
 	}
 
 	w := io.MultiWriter(h, dest)
@@ -148,7 +150,7 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 		return err
 	}
 
-	calculatedHash := stringer.HashToString(h.Sum(nil))
+	calculatedHash := h.HashToString(h.Sum(nil))
 
 	// Check if the hash of the downloaded file matches the expected hash.
 	if calculatedHash != m.Download.Hash {
@@ -157,3 +159,72 @@ func (m Mod) DownloadFile(dest io.Writer) error {
 
 	return nil
 }
+
+// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary
+func (m Mod) GetHashes(hashes []string) (map[string]string, error) {
+	out := make(map[string]string)
+
+	// Get the hash already stored TODO: store multiple (requires breaking pack change)
+	if m.Download.Hash != "" {
+		idx := slices.Index(hashes, m.Download.HashFormat)
+		if idx > -1 {
+			out[m.Download.HashFormat] = m.Download.Hash
+			// Remove hash from list to retrieve
+			hashes = slices.Delete(hashes, idx, idx+1)
+		}
+	}
+
+	// Retrieve the remaining hashes
+	if len(hashes) > 0 {
+		resp, err := http.Get(m.Download.URL)
+		// TODO: content type, user-agent?
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode != 200 {
+			_ = resp.Body.Close()
+			return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
+		}
+
+		// Special fast-path for file length only
+		if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 {
+			out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10)
+			_ = resp.Body.Close()
+			return out, nil
+		}
+
+		mainHasher, err := GetHashImpl(m.Download.HashFormat)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
+		}
+
+		hashers := make([]HashStringer, len(hashes))
+		allHashers := make([]io.Writer, len(hashers))
+		for i, v := range hashes {
+			hashers[i], err = GetHashImpl(v)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err)
+			}
+			allHashers[i] = hashers[i]
+		}
+		allHashers = append(allHashers, mainHasher)
+
+		w := io.MultiWriter(allHashers...)
+		_, err = io.Copy(w, resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download file: %w", err)
+		}
+
+		calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil))
+
+		// Check if the hash of the downloaded file matches the expected hash
+		if calculatedHash != m.Download.Hash {
+			return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
+		}
+
+		for i, v := range hashers {
+			out[hashes[i]] = v.HashToString(v.Sum(nil))
+		}
+	}
+	return out, nil
+}

core/pack.go

@@ -116,7 +116,7 @@ func (pack *Pack) UpdateIndexHash() error {
 	// Hash usage strategy (may change):
 	// Just use SHA256, overwrite existing hash regardless of what it is
 	// May update later to continue using the same hash that was already being used
-	h, stringer, err := GetHashImpl("sha256")
+	h, err := GetHashImpl("sha256")
 	if err != nil {
 		_ = f.Close()
 		return err
@@ -125,7 +125,7 @@ func (pack *Pack) UpdateIndexHash() error {
 		_ = f.Close()
 		return err
 	}
-	hashString := stringer.HashToString(h.Sum(nil))
+	hashString := h.HashToString(h.Sum(nil))
 
 	pack.Index.HashFormat = "sha256"
 	pack.Index.Hash = hashString