WIP: misc fixes, start updating CF/MR export to use download system

core/download.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 )
 
+const DownloadCacheInFolder = "in"
+
 type DownloadSession interface {
 	GetManualDownloads() []ManualDownload
 	StartDownloads() chan CompletedDownload
@@ -20,9 +22,9 @@ type DownloadSession interface {
 }
 
 type CompletedDownload struct {
-	File         *os.File
-	DestFilePath string
-	Hashes       map[string]string
+	File   *os.File
+	Mod    *Mod
+	Hashes map[string]string
 	// Error indicates if/why downloading this file failed
 	Error error
 	// Warnings indicates messages to show to the user regarding this file (download was successful, but had a problem)
@@ -39,7 +41,7 @@ type downloadSessionInternal struct {
 
 type downloadTask struct {
 	metaDownloaderData MetaDownloaderData
-	destFilePath       string
+	mod                *Mod
 	url                string
 	hashFormat         string
 	hash               string
@@ -52,11 +54,23 @@ func (d *downloadSessionInternal) GetManualDownloads() []ManualDownload {
 
 func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload {
 	downloads := make(chan CompletedDownload)
-	for _, task := range d.downloadTasks {
-		// Get handle for mod
-		cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash)
-		if cacheHandle != nil {
-			download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.destFilePath)
+	go func() {
+		for _, task := range d.downloadTasks {
+			// Get handle for mod
+			cacheHandle := d.cacheIndex.GetHandleFromHash(task.hashFormat, task.hash)
+			if cacheHandle != nil {
+				download, err := reuseExistingFile(cacheHandle, d.hashesToObtain, task.mod)
+				if err != nil {
+					downloads <- CompletedDownload{
+						Error: err,
+					}
+				} else {
+					downloads <- download
+				}
+				continue
+			}
+
+			download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex)
 			if err != nil {
 				downloads <- CompletedDownload{
 					Error: err,
@@ -64,18 +78,9 @@ func (d *downloadSessionInternal) StartDownloads() chan CompletedDownload {
 			} else {
 				downloads <- download
 			}
-			continue
 		}
-
-		download, err := downloadNewFile(&task, d.cacheFolder, d.hashesToObtain, &d.cacheIndex)
-		if err != nil {
-			downloads <- CompletedDownload{
-				Error: err,
-			}
-		} else {
-			downloads <- download
-		}
-	}
+		close(downloads)
+	}()
 	return downloads
 }
 
@@ -91,7 +96,7 @@ func (d *downloadSessionInternal) SaveIndex() error {
 	return nil
 }
 
-func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, destFilePath string) (CompletedDownload, error) {
+func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, mod *Mod) (CompletedDownload, error) {
 	// Already stored; try using it!
 	file, err := cacheHandle.Open()
 	if err == nil {
@@ -111,9 +116,9 @@ func reuseExistingFile(cacheHandle *CacheIndexHandle, hashesToObtain []string, d
 		}
 
 		return CompletedDownload{
-			File:         file,
-			DestFilePath: destFilePath,
-			Hashes:       cacheHandle.Hashes,
+			File:   file,
+			Mod:    mod,
+			Hashes: cacheHandle.Hashes,
 		}, nil
 	} else {
 		return CompletedDownload{}, fmt.Errorf("failed to read file %s from cache: %w", cacheHandle.Path(), err)
@@ -151,7 +156,7 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st
 		err = teeHashes(hashesToObtain, hashes, tempFile, data)
 		_ = data.Close()
 		if err != nil {
-			return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.destFilePath, err)
+			return CompletedDownload{}, fmt.Errorf("failed to download file for %s: %w", task.mod.Name, err)
 		}
 	}
 
@@ -180,14 +185,14 @@ func downloadNewFile(task *downloadTask, cacheFolder string, hashesToObtain []st
 	}
 
 	return CompletedDownload{
-		File:         file,
-		DestFilePath: task.destFilePath,
-		Hashes:       hashes,
-		Warnings:     warnings,
+		File:     file,
+		Mod:      task.mod,
+		Hashes:   hashes,
+		Warnings: warnings,
 	}, nil
 }
 
-func selectPreferredHash(hashes map[string]string) (currHash string, currHashFormat string) {
+func selectPreferredHash(hashes map[string]string) (currHashFormat string, currHash string) {
 	for _, hashFormat := range preferredHashList {
 		if hash, ok := hashes[hashFormat]; ok {
 			currHashFormat = hashFormat
@@ -226,7 +231,7 @@ func teeHashes(hashesToObtain []string, hashes map[string]string,
 		return fmt.Errorf("failed to get hash format %s", validateHashFormat)
 	}
 	hashers := make(map[string]HashStringer, len(hashesToObtain))
-	allWriters := make([]io.Writer, len(hashers))
+	allWriters := make([]io.Writer, len(hashesToObtain))
 	for i, v := range hashesToObtain {
 		hashers[v], err = GetHashImpl(v)
 		if err != nil {
@@ -340,11 +345,11 @@ func (h *CacheIndexHandle) CreateFromTemp(temp *os.File) (*os.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = os.Rename(temp.Name(), h.Path())
+	err = temp.Close()
 	if err != nil {
 		return nil, err
 	}
-	err = temp.Close()
+	err = os.Rename(temp.Name(), h.Path())
 	if err != nil {
 		return nil, err
 	}
@@ -425,12 +430,12 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio
 
 	// Get necessary metadata for all files
 	for _, mod := range mods {
-		if mod.Download.Mode == "url" {
+		if mod.Download.Mode == "url" || mod.Download.Mode == "" {
 			downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{
-				destFilePath: mod.GetDestFilePath(),
-				url:          mod.Download.URL,
-				hashFormat:   mod.Download.HashFormat,
-				hash:         mod.Download.Hash,
+				mod:        mod,
+				url:        mod.Download.URL,
+				hashFormat: mod.Download.HashFormat,
+				hash:       mod.Download.Hash,
 			})
 		} else if strings.HasPrefix(mod.Download.Mode, "metadata:") {
 			dlID := strings.TrimPrefix(mod.Download.Mode, "metadata:")
@@ -452,10 +457,11 @@ func CreateDownloadSession(mods []*Mod, hashesToObtain []string) (DownloadSessio
 		for i, v := range mods {
 			isManual, manualDownload := meta[i].GetManualDownload()
 			if isManual {
+				// TODO: lookup in index!
 				downloadSession.manualDownloads = append(downloadSession.manualDownloads, manualDownload)
 			} else {
 				downloadSession.downloadTasks = append(downloadSession.downloadTasks, downloadTask{
-					destFilePath:       v.GetDestFilePath(),
+					mod:                v,
 					metaDownloaderData: meta[i],
 					hashFormat:         v.Download.HashFormat,
 					hash:               v.Download.Hash,

core/index.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -355,6 +356,20 @@ func (in Index) GetAllMods() []string {
 	return list
 }
 
+// LoadAllMods reads all metadata files into Mod structs
+func (in Index) LoadAllMods() ([]*Mod, error) {
+	modPaths := in.GetAllMods()
+	mods := make([]*Mod, len(modPaths))
+	for i, v := range modPaths {
+		modData, err := LoadMod(v)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read metadata file %s: %w", v, err)
+		}
+		mods[i] = &modData
+	}
+	return mods, nil
+}
+
 // GetFilePath attempts to get the path of the destination index file as it is stored on disk
 func (in Index) GetFilePath(f IndexFile) string {
 	return filepath.Join(filepath.Dir(in.indexFile), filepath.FromSlash(f.File))

core/interfaces.go

@@ -51,6 +51,5 @@ type MetaDownloaderData interface {
 type ManualDownload struct {
 	Name     string
 	FileName string
-	DestPath string
 	URL      string
 }

core/mod.go

@@ -2,15 +2,10 @@ package core
 
 import (
 	"errors"
-	"fmt"
-	"golang.org/x/exp/slices"
+	"github.com/BurntSushi/toml"
 	"io"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strconv"
-
-	"github.com/BurntSushi/toml"
 )
 
 // Mod stores metadata about a mod. This is written to a TOML file for each mod.
@@ -134,106 +129,3 @@ func (m Mod) GetFilePath() string {
 func (m Mod) GetDestFilePath() string {
 	return filepath.Join(filepath.Dir(m.metaFile), filepath.FromSlash(m.FileName))
 }
-
-// DownloadFile attempts to resolve and download the file
-func (m Mod) DownloadFile(dest io.Writer) error {
-	// TODO: check mode
-	resp, err := http.Get(m.Download.URL)
-	// TODO: content type, user-agent?
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode != 200 {
-		_ = resp.Body.Close()
-		return errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
-	}
-	h, err := GetHashImpl(m.Download.HashFormat)
-	if err != nil {
-		return fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
-	}
-
-	w := io.MultiWriter(h, dest)
-	_, err = io.Copy(w, resp.Body)
-	if err != nil {
-		return err
-	}
-
-	calculatedHash := h.HashToString(h.Sum(nil))
-
-	// Check if the hash of the downloaded file matches the expected hash.
-	if calculatedHash != m.Download.Hash {
-		return fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
-	}
-
-	return nil
-}
-
-// GetHashes attempts to retrieve the values of all hashes passed to it, downloading if necessary
-func (m Mod) GetHashes(hashes []string) (map[string]string, error) {
-	out := make(map[string]string)
-
-	// Get the hash already stored TODO: store multiple (requires breaking pack change)
-	if m.Download.Hash != "" {
-		idx := slices.Index(hashes, m.Download.HashFormat)
-		if idx > -1 {
-			out[m.Download.HashFormat] = m.Download.Hash
-			// Remove hash from list to retrieve
-			hashes = slices.Delete(hashes, idx, idx+1)
-		}
-	}
-
-	// Retrieve the remaining hashes
-	if len(hashes) > 0 {
-		// TODO: check mode
-		resp, err := http.Get(m.Download.URL)
-		// TODO: content type, user-agent?
-		if err != nil {
-			return nil, err
-		}
-		if resp.StatusCode != 200 {
-			_ = resp.Body.Close()
-			return nil, errors.New("invalid status code " + strconv.Itoa(resp.StatusCode))
-		}
-
-		// Special fast-path for file length only
-		if len(hashes) == 1 && hashes[0] == "length-bytes" && resp.ContentLength > 0 {
-			out["length-bytes"] = strconv.FormatInt(resp.ContentLength, 10)
-			_ = resp.Body.Close()
-			return out, nil
-		}
-
-		mainHasher, err := GetHashImpl(m.Download.HashFormat)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get hash format %s to download file: %w", m.Download.HashFormat, err)
-		}
-
-		hashers := make([]HashStringer, len(hashes))
-		allHashers := make([]io.Writer, len(hashers))
-		for i, v := range hashes {
-			hashers[i], err = GetHashImpl(v)
-			if err != nil {
-				return nil, fmt.Errorf("failed to get hash format %s for file: %w", v, err)
-			}
-			allHashers[i] = hashers[i]
-		}
-		allHashers = append(allHashers, mainHasher)
-
-		w := io.MultiWriter(allHashers...)
-		_, err = io.Copy(w, resp.Body)
-		if err != nil {
-			return nil, fmt.Errorf("failed to download file: %w", err)
-		}
-
-		calculatedHash := mainHasher.HashToString(mainHasher.Sum(nil))
-
-		// Check if the hash of the downloaded file matches the expected hash
-		if calculatedHash != m.Download.Hash {
-			return nil, fmt.Errorf("Hash of downloaded file does not match with expected hash!\n download hash: %s\n expected hash: %s\n", calculatedHash, m.Download.Hash)
-		}
-
-		for i, v := range hashers {
-			out[hashes[i]] = v.HashToString(v.Sum(nil))
-		}
-	}
-	return out, nil
-}