diff --git a/core/hash.go b/core/hash.go index 7e08445..ebd7940 100644 --- a/core/hash.go +++ b/core/hash.go @@ -15,34 +15,76 @@ import ( ) // GetHashImpl gets an implementation of hash.Hash for the given hash type string -func GetHashImpl(hashType string) (hash.Hash, HashStringer, error) { +func GetHashImpl(hashType string) (HashStringer, error) { switch strings.ToLower(hashType) { case "sha1": - return sha1.New(), hexStringer{}, nil + return hexStringer{sha1.New()}, nil case "sha256": - return sha256.New(), hexStringer{}, nil + return hexStringer{sha256.New()}, nil case "sha512": - return sha512.New(), hexStringer{}, nil + return hexStringer{sha512.New()}, nil case "md5": - return md5.New(), hexStringer{}, nil - case "murmur2": - return murmur2.New(), numberStringer{}, nil + return hexStringer{md5.New()}, nil + case "murmur2": // TODO: change to something indicating that this is the CF variant + return number32As64Stringer{murmur2.New()}, nil + case "length-bytes": // TODO: only used internally for now; should not be saved + return number64Stringer{&LengthHasher{}}, nil } - return nil, nil, fmt.Errorf("hash implementation %s not found", hashType) + return nil, fmt.Errorf("hash implementation %s not found", hashType) } type HashStringer interface { + hash.Hash HashToString([]byte) string } -type hexStringer struct{} +type hexStringer struct { + hash.Hash +} func (hexStringer) HashToString(data []byte) string { return hex.EncodeToString(data) } -type numberStringer struct{} +type number32As64Stringer struct { + hash.Hash +} -func (numberStringer) HashToString(data []byte) string { +func (number32As64Stringer) HashToString(data []byte) string { return strconv.FormatUint(uint64(binary.BigEndian.Uint32(data)), 10) } + +type number64Stringer struct { + hash.Hash +} + +func (number64Stringer) HashToString(data []byte) string { + return strconv.FormatUint(binary.BigEndian.Uint64(data), 10) +} + +type LengthHasher struct { + length uint64 +} + +func (h *LengthHasher) Write(p []byte) (n int, err error) { + h.length += uint64(len(p)) + return len(p), nil +} + +func (h *LengthHasher) Sum(b []byte) []byte { + ext := append(b, make([]byte, 8)...) + binary.BigEndian.PutUint64(ext, h.length) + return ext +} + +func (h *LengthHasher) Size() int { + return 8 +} + +func (h *LengthHasher) BlockSize() int { + return 1 +} + +func (h *LengthHasher) Reset() { + h.length = 0 +} diff --git a/core/index.go b/core/index.go index 5777271..9708dd2 100644 --- a/core/index.go +++ b/core/index.go @@ -133,7 +133,7 @@ func (in *Index) updateFile(path string) error { // Hash usage strategy (may change): // Just use SHA256, overwrite existing hash regardless of what it is // May update later to continue using the same hash that was already being used - h, stringer, err := GetHashImpl("sha256") + h, err := GetHashImpl("sha256") if err != nil { _ = f.Close() return err @@ -146,7 +146,7 @@ func (in *Index) updateFile(path string) error { if err != nil { return err } - hashString = stringer.HashToString(h.Sum(nil)) + hashString = h.HashToString(h.Sum(nil)) } mod := false @@ -377,7 +377,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error { if err != nil { return err } - h, stringer, err := GetHashImpl(hashFormat) + h, err := GetHashImpl(hashFormat) if err != nil { return err } @@ -388,7 +388,7 @@ func (in Index) SaveFile(f IndexFile, dest io.Writer) error { return err } - calculatedHash := stringer.HashToString(h.Sum(nil)) + calculatedHash := h.HashToString(h.Sum(nil)) if calculatedHash != f.Hash && !viper.GetBool("no-internal-hashes") { return errors.New("hash of saved file is invalid") } diff --git a/core/mod.go b/core/mod.go index 9afa4e3..61c7fa9 100644 --- a/core/mod.go +++ b/core/mod.go @@ -3,6 +3,7 @@ package core import ( "errors" "fmt" + "golang.org/x/exp/slices" "io" "net/http" "os" @@ -92,7 +93,7 @@ func (m Mod) Write() (string, string, error) { } } - h, stringer, err := GetHashImpl("sha256") + h, err := GetHashImpl("sha256") if err != nil { _ = f.Close() return "", "", err @@ -103,7 +104,7 @@ func (m Mod) Write() (string, string, error) { // Disable indentation enc.Indent = "" err = enc.Encode(m) - hashString := stringer.HashToString(h.Sum(nil)) + hashString := h.HashToString(h.Sum(nil)) if err != nil { _ = f.Close() return "sha256", hashString, err @@ -130,6 +131,7 @@ func (m Mod) GetDestFilePath() string { // DownloadFile attempts to resolve and download the file func (m Mod) DownloadFile(dest io.Writer) error { resp, err := http.Get(m.Download.URL) + // TODO: content type, user-agent? if err != nil { return err } @@ -137,9 +139,9 @@ func (m Mod) DownloadFile(dest io.Writer) error { _ = resp.Body.Close() return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode)) } - h, stringer, err := GetHashImpl(m.Download.HashFormat) + h, err := GetHashImpl(m.Download.HashFormat) if err != nil { - return err + return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err) } w := io.MultiWriter(h, dest) @@ -148,7 +150,7 @@ func (m Mod) DownloadFile(dest io.Writer) error { return err } - calculatedHash := stringer.HashToString(h.Sum(nil)) + calculatedHash := h.HashToString(h.Sum(nil)) // Check if the hash of the downloaded file matches the expected hash. if calculatedHash != m.Download.Hash { @@ -157,3 +159,72 @@ func (m Mod) DownloadFile(dest io.Writer) error { return nil } + +// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary +func (m Mod) GetHashes(hashes []string) (map[string]string, error) { + out := make(map[string]string) + + // Get the hash already stored TODO: store multiple (requires breaking pack change) + if m.Download.Hash != "" { + idx := slices.Index(hashes, m.Download.HashFormat) + if idx > -1 { + out[m.Download.HashFormat] = m.Download.Hash + // Remove hash from list to retrieve + hashes = slices.Delete(hashes, idx, idx+1) + } + } + + // Retrieve the remaining hashes + if len(hashes) > 0 { + resp, err := http.Get(m.Download.URL) + // TODO: content type, user-agent? + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + _ = resp.Body.Close() + return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode)) + } + + // Special fast-path for file length only + if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 { + out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10) + _ = resp.Body.Close() + return out, nil + } + + mainHasher, err := GetHashImpl(m.Download.HashFormat) + if err != nil { + return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err) + } + + hashers := make([]HashStringer, len(hashes)) + allHashers := make([]io.Writer, len(hashers)) + for i, v := range hashes { + hashers[i], err = GetHashImpl(v) + if err != nil { + return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err) + } + allHashers[i] = hashers[i] + } + allHashers = append(allHashers, mainHasher) + + w := io.MultiWriter(allHashers...) + _, err = io.Copy(w, resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to download file: %w", err) + } + + calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil)) + + // Check if the hash of the downloaded file matches the expected hash + if calculatedHash != m.Download.Hash { + return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash) + } + + for i, v := range hashers { + out[hashes[i]] = v.HashToString(v.Sum(nil)) + } + } + return out, nil +} diff --git a/core/pack.go b/core/pack.go index 5db7a06..b0bee5e 100644 --- a/core/pack.go +++ b/core/pack.go @@ -116,7 +116,7 @@ func (pack *Pack) UpdateIndexHash() error { // Hash usage strategy (may change): // Just use SHA256, overwrite existing hash regardless of what it is // May update later to continue using the same hash that was already being used - h, stringer, err := GetHashImpl("sha256") + h, err := GetHashImpl("sha256") if err != nil { _ = f.Close() return err @@ -125,7 +125,7 @@ func (pack *Pack) UpdateIndexHash() error { _ = f.Close() return err } - hashString := stringer.HashToString(h.Sum(nil)) + hashString := h.HashToString(h.Sum(nil)) pack.Index.HashFormat = "sha256" pack.Index.Hash = hashString diff --git a/modrinth/export.go b/modrinth/export.go index 25c48a9..009589f 100644 --- a/modrinth/export.go +++ b/modrinth/export.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" @@ -77,26 +78,15 @@ var exportCmd = &cobra.Command{ } // TODO: cache these (ideally with changes to pack format) - fmt.Println("Retrieving SHA1 hashes for external mods...") - sha1Hashes := make([]string, len(mods)) + fmt.Println("Retrieving hashes for external mods...") + modsHashes := make([]map[string]string, len(mods)) for i, mod := range mods { - if mod.Download.HashFormat == "sha1" { - sha1Hashes[i] = mod.Download.Hash - } else { - // Hash format for this mod isn't SHA1 - and the Modrinth pack format requires it; so get it by downloading the file - h, stringer, err := core.GetHashImpl("sha1") - if err != nil { - panic("Failed to get sha1 hash implementation") - } - err = mod.DownloadFile(h) - if err != nil { - fmt.Printf("Error downloading mod file %s: %s\n", mod.Download.URL, err.Error()) - // TODO: exit(1)? - continue - } - sha1Hashes[i] = stringer.HashToString(h.Sum(nil)) - fmt.Printf("Retrieved SHA1 hash for %s successfully\n", mod.Download.URL) + modsHashes[i], err = mod.GetHashes([]string{"sha1", "sha512", "length-bytes"}) + if err != nil { + fmt.Printf("Error downloading mod file %s: %s\n", mod.Download.URL, err.Error()) + continue } + fmt.Printf("Retrieved hashes for %s successfully\n", mod.Download.URL) } manifestFile, err := exp.Create("modrinth.index.json") @@ -119,7 +109,12 @@ var exportCmd = &cobra.Command{ path := filepath.ToSlash(pathForward) hashes := make(map[string]string) - hashes["sha1"] = sha1Hashes[i] + hashes["sha1"] = modsHashes[i]["sha1"] + hashes["sha512"] = modsHashes[i]["sha512"] + fileSize, err := strconv.ParseUint(modsHashes[i]["length-bytes"], 10, 64) + if err != nil { + panic(err) + } // Create env options based on configured optional/side var envInstalled string @@ -155,6 +150,7 @@ var exportCmd = &cobra.Command{ Server string `json:"server"` }{Client: clientEnv, Server: serverEnv}, Downloads: []string{u}, + FileSize: uint32(fileSize), } } diff --git a/modrinth/pack.go b/modrinth/pack.go index 01ef051..02050dd 100644 --- a/modrinth/pack.go +++ b/modrinth/pack.go @@ -18,4 +18,5 @@ type PackFile struct { Server string `json:"server"` } `json:"env"` Downloads []string `json:"downloads"` + FileSize uint32 `json:"fileSize"` }