Automatically download and bundle unwhitelisted mods/files in Modrinth modpacks

.github/workflows/go.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.16
+    - name: Set up Go 1.18
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.18
       id: go
 
     - name: Check out code into the Go module directory

go.mod

@@ -19,13 +19,15 @@ require (
 	github.com/spf13/viper v1.8.0
 	github.com/vbauerster/mpb/v4 v4.12.2
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/dixonwille/wlog.v2 v2.0.0 // indirect
 	gopkg.in/dixonwille/wmenu.v4 v4.0.2
 )
 
+require golang.org/x/exp v0.0.0-20220328175248-053ad81199eb
+
 require (
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
@@ -43,4 +45,4 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
-go 1.17
+go 1.18

go.sum

@@ -374,6 +374,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220328175248-053ad81199eb h1:pC9Okm6BVmxEw76PUu0XUbOTQ92JX11hfvqTjAV3qxM=
+golang.org/x/exp v0.0.0-20220328175248-053ad81199eb/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -506,9 +508,8 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

modrinth/export.go

@@ -5,11 +5,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/viper"
+	"net/url"
 	"os"
 	"path/filepath"
 
 	"github.com/packwiz/packwiz/core"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 // exportCmd represents the export command
@@ -54,7 +56,7 @@ var exportCmd = &cobra.Command{
 		// TODO: should index just expose indexPath itself, through a function?
 		indexPath := filepath.Join(filepath.Dir(viper.GetString("pack-file")), filepath.FromSlash(pack.Index.File))
 
-		mods := loadMods(index)
+		mods, unwhitelistedMods := loadMods(index)
 
 		fileName := viper.GetString("modrinth.export.output")
 		if fileName == "" {
@@ -218,6 +220,40 @@ var exportCmd = &cobra.Command{
 			}
 		}
 
+		if len(unwhitelistedMods) > 0 {
+			fmt.Println("Downloading unwhitelisted mods...")
+		}
+		for _, v := range unwhitelistedMods {
+			pathRel, err := filepath.Rel(filepath.Dir(indexPath), v.GetDestFilePath())
+			if err != nil {
+				fmt.Printf("Error resolving mod file: %s\n", err.Error())
+				// TODO: exit(1)?
+				continue
+			}
+			var path string
+			if v.Side == core.ClientSide {
+				path = filepath.ToSlash(filepath.Join("client-overrides", pathRel))
+			} else if v.Side == core.ServerSide {
+				path = filepath.ToSlash(filepath.Join("server-overrides", pathRel))
+			} else {
+				path = filepath.ToSlash(filepath.Join("overrides", pathRel))
+			}
+
+			file, err := exp.Create(path)
+			if err != nil {
+				fmt.Printf("Error creating file: %s\n", err.Error())
+				// TODO: exit(1)?
+				continue
+			}
+			err = v.DownloadFile(file)
+			if err != nil {
+				fmt.Printf("Error downloading file: %s\n", err.Error())
+				// TODO: exit(1)?
+				continue
+			}
+			fmt.Printf("Downloaded %v successfully\n", path)
+		}
+
 		err = exp.Close()
 		if err != nil {
 			fmt.Println("Error writing export file: " + err.Error())
@@ -233,10 +269,17 @@ var exportCmd = &cobra.Command{
 	},
 }
 
-func loadMods(index core.Index) []core.Mod {
+var whitelistedHosts = []string{
+	"cdn.modrinth.com",
+	"edge.forgecdn.net",
+	"github.com",
+	"raw.githubusercontent.com",
+}
+
+func loadMods(index core.Index) ([]core.Mod, []core.Mod) {
 	modPaths := index.GetAllMods()
-	mods := make([]core.Mod, len(modPaths))
+	mods := make([]core.Mod, 0, len(modPaths))
-	i := 0
+	unwhitelistedMods := make([]core.Mod, 0)
 	fmt.Println("Reading mod files...")
 	for _, v := range modPaths {
 		modData, err := core.LoadMod(v)
@@ -246,10 +289,19 @@ func loadMods(index core.Index) []core.Mod {
 			continue
 		}
 
-		mods[i] = modData
+		modUrl, err := url.Parse(modData.Download.URL)
-		i++
+		if err == nil {
+			if slices.Contains(whitelistedHosts, modUrl.Host) {
+				mods = append(mods, modData)
+			} else {
+				unwhitelistedMods = append(unwhitelistedMods, modData)
 			}
-	return mods[:i]
+		} else {
+			fmt.Printf("Failed to parse mod URL: %v\n", modUrl)
+			mods = append(mods, modData)
+		}
+	}
+	return mods, unwhitelistedMods
 }
 
 func init() {