Change backend request code to use new CurseForge API (WIP)

See the packwiz Discord for more information, as the changes with the new API Terms and Conditions have some implications for packwiz.
This commit isn't fully functional yet; I have more changes to make.

curseforge/curseforge.go

@@ -176,11 +176,6 @@ func createModFile(modInfo modInfo, fileInfo modFileInfo, index *core.Index, opt
 		return err
 	}
 
-	u, err := core.ReencodeURL(fileInfo.DownloadURL)
-	if err != nil {
-		return err
-	}
-
 	hash, hashFormat := fileInfo.getBestHash()
 
 	var optional *core.ModOption
@@ -196,7 +191,6 @@ func createModFile(modInfo modInfo, fileInfo modFileInfo, index *core.Index, opt
 		FileName: fileInfo.FileName,
 		Side:     core.UniversalSide,
 		Download: core.ModDownload{
-			URL:        u,
 			HashFormat: hashFormat,
 			Hash:       hash,
 		},
@@ -335,7 +329,7 @@ func (u cfUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack
 		modIDs[i] = project.ProjectID
 	}
 
-	modInfosUnsorted, err := getModInfoMultiple(modIDs)
+	modInfosUnsorted, err := cfDefaultClient.getModInfoMultiple(modIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -420,22 +414,16 @@ func (u cfUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error {
 		fileInfoData := modState.fileInfo
 		if !modState.hasFileInfo {
 			var err error
-			fileInfoData, err = getFileInfo(modState.ID, modState.fileID)
+			fileInfoData, err = cfDefaultClient.getFileInfo(modState.ID, modState.fileID)
 			if err != nil {
 				return err
 			}
 		}
 
-		u, err := core.ReencodeURL(fileInfoData.DownloadURL)
-		if err != nil {
-			return err
-		}
-
 		v.FileName = fileInfoData.FileName
 		v.Name = modState.Name
 		hash, hashFormat := fileInfoData.getBestHash()
 		v.Download = core.ModDownload{
-			URL:        u,
 			HashFormat: hashFormat,
 			Hash:       hash,
 		}

curseforge/detect.go

@@ -61,7 +61,7 @@ var detectCmd = &cobra.Command{
 		}
 		fmt.Printf("Found %d files, submitting...\n", len(hashes))
 
-		res, err := getFingerprintInfo(hashes)
+		res, err := cfDefaultClient.getFingerprintInfo(hashes)
 		if err != nil {
 			fmt.Println(err)
 			return
@@ -82,7 +82,7 @@ var detectCmd = &cobra.Command{
 		}
 		fmt.Println("Installing...")
 		for _, v := range res.ExactMatches {
-			modInfoData, err := getModInfo(v.ID)
+			modInfoData, err := cfDefaultClient.getModInfo(v.ID)
 			if err != nil {
 				fmt.Println(err)
 				os.Exit(1)

curseforge/import.go

@@ -192,7 +192,7 @@ var importCmd = &cobra.Command{
 
 		fmt.Println("Querying Curse API for mod info...")
 
-		modInfos, err := getModInfoMultiple(modIDs)
+		modInfos, err := cfDefaultClient.getModInfoMultiple(modIDs)
 		if err != nil {
 			fmt.Printf("Failed to obtain mod information: %s\n", err)
 			os.Exit(1)
@@ -236,16 +236,14 @@ var importCmd = &cobra.Command{
 		// 2nd pass: query files that weren't in the previous results
 		fmt.Println("Querying Curse API for file info...")
 
-		modFileInfos, err := getFileInfoMultiple(remainingFileIDs)
+		modFileInfos, err := cfDefaultClient.getFileInfoMultiple(remainingFileIDs)
 		if err != nil {
 			fmt.Printf("Failed to obtain mod file information: %s\n", err)
 			os.Exit(1)
 		}
 
 		for _, v := range modFileInfos {
-			for _, file := range v {
+			modFileInfosMap[v.ID] = v
-				modFileInfosMap[file.ID] = file
-			}
 		}
 
 		// 3rd pass: create mod files for every file

curseforge/install.go

@@ -99,7 +99,7 @@ var installCmd = &cobra.Command{
 		}
 
 		if !modInfoObtained {
-			modInfoData, err = getModInfo(modID)
+			modInfoData, err = cfDefaultClient.getModInfo(modID)
 			if err != nil {
 				fmt.Println(err)
 				os.Exit(1)
@@ -169,7 +169,7 @@ var installCmd = &cobra.Command{
 					}
 					depIDPendingQueue = depIDPendingQueue[:i]
 
-					depInfoData, err := getModInfoMultiple(depIDPendingQueue)
+					depInfoData, err := cfDefaultClient.getModInfoMultiple(depIDPendingQueue)
 					if err != nil {
 						fmt.Printf("Error retrieving dependency data: %s\n", err.Error())
 					}
@@ -276,7 +276,7 @@ func searchCurseforgeInternal(args []string, mcVersion string, packLoaderType in
 	if len(viper.GetStringSlice("acceptable-game-versions")) > 0 {
 		filterGameVersion = ""
 	}
-	results, err := getSearch(searchTerm, filterGameVersion, packLoaderType)
+	results, err := cfDefaultClient.getSearch(searchTerm, filterGameVersion, packLoaderType)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -373,7 +373,7 @@ func getLatestFile(modInfoData modInfo, mcVersion string, fileID int, packLoader
 		}
 	}
 
-	fileInfoData, err := getFileInfo(modInfoData.ID, fileID)
+	fileInfoData, err := cfDefaultClient.getFileInfo(modInfoData.ID, fileID)
 	if err != nil {
 		return modFileInfo{}, err
 	}

curseforge/request.go

@@ -2,6 +2,7 @@ package curseforge
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -9,10 +10,84 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
-	"strings"
 	"time"
 )
 
+// TODO: update everything for no URL and download mode "metadata:curseforge"
+
+const cfApiServer = "api.curseforge.com"
+
+// If you fork/derive from packwiz, I request that you obtain your own API key.
+const cfApiKeyDefault = "JDJhJDEwJHNBWVhqblU1N0EzSmpzcmJYM3JVdk92UWk2NHBLS3BnQ2VpbGc1TUM1UGNKL0RYTmlGWWxh"
+
+// Exists so you can provide it as a build parameter: -ldflags="-X 'github.com/packwiz/packwiz/curseforge.cfApiKey=key'"
+var cfApiKey = ""
+
+func decodeDefaultKey() string {
+	k, err := base64.StdEncoding.DecodeString(cfApiKeyDefault)
+	if err != nil {
+		panic("failed to read API key!")
+	}
+	return string(k)
+}
+
+type cfApiClient struct {
+	httpClient *http.Client
+}
+
+var cfDefaultClient = cfApiClient{&http.Client{}}
+
+func (c *cfApiClient) makeGet(endpoint string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", "https://"+cfApiServer+endpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: make this configurable application-wide
+	req.Header.Set("User-Agent", "packwiz/packwiz client")
+	req.Header.Set("Accept", "application/json")
+	if cfApiKey == "" {
+		cfApiKey = decodeDefaultKey()
+	}
+	req.Header.Set("X-API-Key", cfApiKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid response status: %v", resp.Status)
+	}
+	return resp, nil
+}
+
+func (c *cfApiClient) makePost(endpoint string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest("POST", "https://"+cfApiServer+endpoint, body)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: make this configurable application-wide
+	req.Header.Set("User-Agent", "packwiz/packwiz client")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+	if cfApiKey == "" {
+		cfApiKey = decodeDefaultKey()
+	}
+	req.Header.Set("X-API-Key", cfApiKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid response status: %v", resp.Status)
+	}
+	return resp, nil
+}
+
 // addonSlugRequest is sent to the CurseProxy GraphQL api to get the id from a slug
 type addonSlugRequest struct {
 	Query     string `json:"query"`
@@ -64,6 +139,7 @@ func modIDFromSlug(slug string) (int, error) {
 		return 0, err
 	}
 
+	// TODO: move to new slug API
 	req, err := http.NewRequest("POST", "https://curse.nikky.moe/graphql", bytes.NewBuffer(requestBytes))
 	if err != nil {
 		return 0, err
@@ -134,129 +210,91 @@ const (
 type modInfo struct {
 	Name                   string        `json:"name"`
 	Slug                   string        `json:"slug"`
-	WebsiteURL             string        `json:"websiteUrl"`
 	ID                     int           `json:"id"`
 	LatestFiles            []modFileInfo `json:"latestFiles"`
 	GameVersionLatestFiles []struct {
 		// TODO: check how twitch launcher chooses which one to use, when you are on beta/alpha channel?!
 		// or does it not have the concept of release channels?!
 		GameVersion string `json:"gameVersion"`
-		ID          int    `json:"projectFileId"`
+		ID          int    `json:"fileId"`
-		Name        string `json:"projectFileName"`
+		Name        string `json:"filename"`
-		FileType    int    `json:"fileType"`
+		FileType    int    `json:"releaseType"`
 		Modloader   int    `json:"modLoader"`
-	} `json:"gameVersionLatestFiles"`
+	} `json:"latestFilesIndexes"`
 	ModLoaders []string `json:"modLoaders"`
 }
 
-func getModInfo(modID int) (modInfo, error) {
+func (c *cfApiClient) getModInfo(modID int) (modInfo, error) {
-	var infoRes modInfo
+	var infoRes struct {
-	client := &http.Client{}
+		Data modInfo `json:"data"`
+	}
 
 	idStr := strconv.Itoa(modID)
-
+	resp, err := c.makeGet("/v1/mods/" + idStr)
-	req, err := http.NewRequest("GET", "https://addons-ecs.forgesvc.net/api/v2/addon/"+idStr, nil)
 	if err != nil {
-		return modInfo{}, err
+		return modInfo{}, fmt.Errorf("failed to request addon data for ID %d: %w", modID, err)
-	}
-
-	// TODO: make this configurable application-wide
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return modInfo{}, err
-	}
-	if resp.StatusCode != 200 {
-		return modInfo{}, fmt.Errorf("failed to request addon ID %d: %s", modID, resp.Status)
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
 	if err != nil && err != io.EOF {
-		return modInfo{}, err
+		return modInfo{}, fmt.Errorf("failed to request addon data for ID %d: %w", modID, err)
 	}
 
-	if infoRes.ID != modID {
+	if infoRes.Data.ID != modID {
-		return modInfo{}, fmt.Errorf("unexpected addon ID in CurseForge response: %d (expected %d)", infoRes.ID, modID)
+		return modInfo{}, fmt.Errorf("unexpected addon ID in CurseForge response: %d (expected %d)", infoRes.Data.ID, modID)
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
 }
 
-func getModInfoMultiple(modIDs []int) ([]modInfo, error) {
+func (c *cfApiClient) getModInfoMultiple(modIDs []int) ([]modInfo, error) {
-	var infoRes []modInfo
+	var infoRes struct {
-	client := &http.Client{}
+		Data []modInfo `json:"data"`
+	}
 
-	modIDsData, err := json.Marshal(modIDs)
+	modIDsData, err := json.Marshal(struct {
+		ModIDs []int `json:"modIds"`
+	}{
+		ModIDs: modIDs,
+	})
 	if err != nil {
 		return []modInfo{}, err
 	}
 
-	req, err := http.NewRequest("POST", "https://addons-ecs.forgesvc.net/api/v2/addon/", bytes.NewBuffer(modIDsData))
+	resp, err := c.makePost("/v1/mods", bytes.NewBuffer(modIDsData))
 	if err != nil {
-		return []modInfo{}, err
+		return []modInfo{}, fmt.Errorf("failed to request addon data: %w", err)
-	}
-
-	// TODO: make this configurable application-wide
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return []modInfo{}, err
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
 	if err != nil && err != io.EOF {
-		return []modInfo{}, err
+		return []modInfo{}, fmt.Errorf("failed to request addon data: %w", err)
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
-}
-
-const cfDateFormatString = "2006-01-02T15:04:05.999"
-
-type cfDateFormat struct {
-	time.Time
-}
-
-// Curse switched to proper RFC3339, but previously downloaded metadata still uses the old format :(
-func (f *cfDateFormat) UnmarshalJSON(input []byte) error {
-	trimmed := strings.Trim(string(input), `"`)
-	timeValue, err := time.Parse(time.RFC3339Nano, trimmed)
-	if err != nil {
-		timeValue, err = time.Parse(cfDateFormatString, trimmed)
-		if err != nil {
-			return err
-		}
-	}
-
-	f.Time = timeValue
-	return nil
 }
 
 // modFileInfo is a subset of the deserialised JSON response from the Curse API for mod files
 type modFileInfo struct {
-	ID           int          `json:"id"`
+	ID           int       `json:"id"`
-	FileName     string       `json:"fileName"`
+	FileName     string    `json:"fileName"`
-	FriendlyName string       `json:"displayName"`
+	FriendlyName string    `json:"displayName"`
-	Date         cfDateFormat `json:"fileDate"`
+	Date         time.Time `json:"fileDate"`
-	Length       int          `json:"fileLength"`
+	Length       int       `json:"fileLength"`
-	FileType     int          `json:"releaseType"`
+	FileType     int       `json:"releaseType"`
 	// fileStatus? means latest/preferred?
+	// According to the CurseForge API T&Cs, this must not be saved or cached
 	DownloadURL  string   `json:"downloadUrl"`
-	GameVersions []string `json:"gameVersion"`
+	GameVersions []string `json:"gameVersions"`
-	Fingerprint  int      `json:"packageFingerprint"`
+	Fingerprint  int      `json:"fileFingerprint"`
 	Dependencies []struct {
-		ModID int `json:"addonId"`
+		ModID int `json:"modId"`
-		Type  int `json:"type"`
+		Type  int `json:"relationType"`
 	} `json:"dependencies"`
 
 	Hashes []struct {
 		Value     string `json:"value"`
-		Algorithm int    `json:"algorithm"`
+		Algorithm int    `json:"algo"`
 	} `json:"hashes"`
 }
 
@@ -286,105 +324,78 @@ func (i modFileInfo) getBestHash() (hash string, hashFormat string) {
 	return
 }
 
-func getFileInfo(modID int, fileID int) (modFileInfo, error) {
+func (c *cfApiClient) getFileInfo(modID int, fileID int) (modFileInfo, error) {
-	var infoRes modFileInfo
+	var infoRes struct {
-	client := &http.Client{}
+		Data modFileInfo `json:"data"`
+	}
 
 	modIDStr := strconv.Itoa(modID)
 	fileIDStr := strconv.Itoa(fileID)
 
-	req, err := http.NewRequest("GET", "https://addons-ecs.forgesvc.net/api/v2/addon/"+modIDStr+"/file/"+fileIDStr, nil)
+	resp, err := c.makeGet("/v1/mods/" + modIDStr + "/files/" + fileIDStr)
 	if err != nil {
-		return modFileInfo{}, err
+		return modFileInfo{}, fmt.Errorf("failed to request file data for addon ID %d, file ID %d: %w", modID, fileID, err)
-	}
-
-	// TODO: make this configurable application-wide
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return modFileInfo{}, err
-	}
-	if resp.StatusCode != 200 {
-		return modFileInfo{}, fmt.Errorf("failed to request file ID %d for addon %d: %s", fileID, modID, resp.Status)
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
 	if err != nil && err != io.EOF {
-		return modFileInfo{}, err
+		return modFileInfo{}, fmt.Errorf("failed to request file data for addon ID %d, file ID %d: %w", modID, fileID, err)
 	}
 
-	if infoRes.ID != fileID {
+	if infoRes.Data.ID != fileID {
-		return modFileInfo{}, fmt.Errorf("unexpected file ID for addon %d in CurseForge response: %d (expected %d)", modID, infoRes.ID, fileID)
+		return modFileInfo{}, fmt.Errorf("unexpected file ID for addon %d in CurseForge response: %d (expected %d)", modID, infoRes.Data.ID, fileID)
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
 }
 
-func getFileInfoMultiple(fileIDs []int) (map[string][]modFileInfo, error) {
+func (c *cfApiClient) getFileInfoMultiple(fileIDs []int) ([]modFileInfo, error) {
-	var infoRes map[string][]modFileInfo
+	var infoRes struct {
-	client := &http.Client{}
+		Data []modFileInfo `json:"data"`
-
-	modIDsData, err := json.Marshal(fileIDs)
-	if err != nil {
-		return make(map[string][]modFileInfo), err
 	}
 
-	req, err := http.NewRequest("POST", "https://addons-ecs.forgesvc.net/api/v2/addon/files", bytes.NewBuffer(modIDsData))
+	fileIDsData, err := json.Marshal(struct {
+		FileIDs []int `json:"fileIds"`
+	}{
+		FileIDs: fileIDs,
+	})
 	if err != nil {
-		return make(map[string][]modFileInfo), err
+		return []modFileInfo{}, err
 	}
 
-	// TODO: make this configurable application-wide
+	resp, err := c.makePost("/v1/mods/files", bytes.NewBuffer(fileIDsData))
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := client.Do(req)
 	if err != nil {
-		return make(map[string][]modFileInfo), err
+		return []modFileInfo{}, fmt.Errorf("failed to request file data: %w", err)
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
 	if err != nil && err != io.EOF {
-		return make(map[string][]modFileInfo), err
+		return []modFileInfo{}, fmt.Errorf("failed to request file data: %w", err)
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
 }
 
-func getSearch(searchText string, gameVersion string, modloaderType int) ([]modInfo, error) {
+func (c *cfApiClient) getSearch(searchText string, gameVersion string, modloaderType int) ([]modInfo, error) {
-	var infoRes []modInfo
+	var infoRes struct {
-	client := &http.Client{}
+		Data []modInfo `json:"data"`
-
-	reqURL, err := url.Parse("https://addons-ecs.forgesvc.net/api/v2/addon/search?gameId=432&pageSize=10&categoryId=0&sectionId=6")
-	if err != nil {
-		return []modInfo{}, err
 	}
-	q := reqURL.Query()
+
+	q := url.Values{}
+	q.Set("gameId", "432") // Minecraft
+	q.Set("pageSize", "10")
+	q.Set("classId", "6") // Mods
 	q.Set("searchFilter", searchText)
-
 	if len(gameVersion) > 0 {
 		q.Set("gameVersion", gameVersion)
 	}
 	if modloaderType != modloaderTypeAny {
 		q.Set("modLoaderType", strconv.Itoa(modloaderType))
 	}
-	reqURL.RawQuery = q.Encode()
 
-	req, err := http.NewRequest("GET", reqURL.String(), nil)
+	resp, err := c.makeGet("/v1/mods/search?" + q.Encode())
 	if err != nil {
-		return []modInfo{}, err
+		return []modInfo{}, fmt.Errorf("failed to retrieve search results: %w", err)
-	}
-
-	// TODO: make this configurable application-wide
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return []modInfo{}, err
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
@@ -392,7 +403,7 @@ func getSearch(searchText string, gameVersion string, modloaderType int) ([]modI
 		return []modInfo{}, err
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
 }
 
 type addonFingerprintResponse struct {
@@ -409,28 +420,23 @@ type addonFingerprintResponse struct {
 	UnmatchedFingerprints    []int    `json:"unmatchedFingerprints"`
 }
 
-func getFingerprintInfo(hashes []int) (addonFingerprintResponse, error) {
+func (c *cfApiClient) getFingerprintInfo(hashes []int) (addonFingerprintResponse, error) {
-	var infoRes addonFingerprintResponse
+	var infoRes struct {
-	client := &http.Client{}
+		Data addonFingerprintResponse `json:"data"`
+	}
 
-	hashesData, err := json.Marshal(hashes)
+	hashesData, err := json.Marshal(struct {
+		Fingerprints []int `json:"fingerprints"`
+	}{
+		Fingerprints: hashes,
+	})
 	if err != nil {
 		return addonFingerprintResponse{}, err
 	}
 
-	req, err := http.NewRequest("POST", "https://addons-ecs.forgesvc.net/api/v2/fingerprint", bytes.NewBuffer(hashesData))
+	resp, err := c.makePost("/v1/fingerprints", bytes.NewBuffer(hashesData))
 	if err != nil {
-		return addonFingerprintResponse{}, err
+		return addonFingerprintResponse{}, fmt.Errorf("failed to retrieve fingerprint results: %w", err)
-	}
-
-	// TODO: make this configurable application-wide
-	req.Header.Set("User-Agent", "packwiz/packwiz client")
-	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return addonFingerprintResponse{}, err
 	}
 
 	err = json.NewDecoder(resp.Body).Decode(&infoRes)
@@ -438,5 +444,5 @@ func getFingerprintInfo(hashes []int) (addonFingerprintResponse, error) {
 		return addonFingerprintResponse{}, err
 	}
 
-	return infoRes, nil
+	return infoRes.Data, nil
 }