Merge branch 'main' into pr/unascribed/258

core/download.go

@@ -4,12 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"golang.org/x/exp/slices"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/exp/slices"
 )
 
 const UserAgent = "packwiz/packwiz"

github/github.go

@@ -0,0 +1,103 @@
+package github
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/packwiz/packwiz/cmd"
+	"github.com/packwiz/packwiz/core"
+	"github.com/spf13/cobra"
+)
+
+var githubCmd = &cobra.Command{
+	Use:     "github",
+	Aliases: []string{"gh"},
+	Short:   "Manage projects released on GitHub",
+}
+
+func init() {
+	cmd.Add(githubCmd)
+	core.Updaters["github"] = ghUpdater{}
+}
+
+func fetchRepo(slug string) (Repo, error) {
+	var repo Repo
+
+	res, err := ghDefaultClient.getRepo(slug)
+	if err != nil {
+		return repo, err
+	}
+
+	defer res.Body.Close()
+
+	repoBody, err := io.ReadAll(res.Body)
+	if err != nil {
+		return repo, err
+	}
+
+	err = json.Unmarshal(repoBody, &repo)
+	if err != nil {
+		return repo, err
+	}
+
+	if repo.FullName == "" {
+		return repo, errors.New("invalid json while fetching project: " + slug)
+	}
+
+	return repo, nil
+}
+
+type Repo struct {
+	ID       int    `json:"id"`
+	Name     string `json:"name"`      // "hello_world"
+	FullName string `json:"full_name"` // "owner/hello_world"
+}
+
+type Release struct {
+	URL             string  `json:"url"`
+	TagName         string  `json:"tag_name"`
+	TargetCommitish string  `json:"target_commitish"` // The branch of the release
+	Name            string  `json:"name"`
+	CreatedAt       string  `json:"created_at"`
+	Assets          []Asset `json:"assets"`
+}
+
+type Asset struct {
+	URL                string `json:"url"`
+	BrowserDownloadURL string `json:"browser_download_url"`
+	Name               string `json:"name"`
+}
+
+func (u ghUpdateData) ToMap() (map[string]interface{}, error) {
+	newMap := make(map[string]interface{})
+	err := mapstructure.Decode(u, &newMap)
+	return newMap, err
+}
+
+func (u Asset) getSha256() (string, error) {
+	// TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github!
+	mainHasher, err := core.GetHashImpl("sha256")
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := ghDefaultClient.makeGet(u.BrowserDownloadURL)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	mainHasher.Write(body)
+
+	hash := mainHasher.Sum(nil)
+
+	return mainHasher.HashToString(hash), nil
+}

github/install.go

@@ -0,0 +1,232 @@
+package github
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/dlclark/regexp2"
+	"github.com/packwiz/packwiz/core"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var GithubRegex = regexp.MustCompile(`^https?://(?:www\.)?github\.com/([^/]+/[^/]+)`)
+
+// installCmd represents the install command
+var installCmd = &cobra.Command{
+	Use:     "add [URL|slug]",
+	Short:   "Add a project from a GitHub repository URL or slug",
+	Aliases: []string{"install", "get"},
+	Args:    cobra.ArbitraryArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		pack, err := core.LoadPack()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		if len(args) == 0 || len(args[0]) == 0 {
+			fmt.Println("You must specify a GitHub repository URL.")
+			os.Exit(1)
+		}
+
+		// Try interpreting the argument as a slug, or GitHub repository URL.
+		var slug string
+		var branch string
+
+		// Regex to match potential release assets against.
+		// The default will match any asset with a name that does *not* end with:
+		// - "-api.jar"
+		// - "-dev.jar"
+		// - "-dev-preshadow.jar"
+		// - "-sources.jar"
+		// In most cases, this will only match one asset.
+		// TODO: Hopefully.
+		regex := `^.+(?<!-api|-dev|-dev-preshadow|-sources)\.jar$`
+
+		// Check if the argument is a valid GitHub repository URL; if so, extract the slug from the URL.
+		// Otherwise, interpret the argument as a slug directly.
+		matches := GithubRegex.FindStringSubmatch(args[0])
+		if len(matches) == 2 {
+			slug = matches[1]
+		} else {
+			slug = args[0]
+		}
+
+		repo, err := fetchRepo(slug)
+
+		if err != nil {
+			fmt.Printf("Failed to add project: %s\n", err)
+			os.Exit(1)
+		}
+
+		if branchFlag != "" {
+			branch = branchFlag
+		}
+		if regexFlag != "" {
+			regex = regexFlag
+		}
+
+		err = installMod(repo, branch, regex, pack)
+		if err != nil {
+			fmt.Printf("Failed to add project: %s\n", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func installMod(repo Repo, branch string, regex string, pack core.Pack) error {
+	latestRelease, err := getLatestRelease(repo.FullName, branch)
+	if err != nil {
+		return fmt.Errorf("failed to get latest release: %v", err)
+	}
+
+	return installRelease(repo, latestRelease, regex, pack)
+}
+
+func getLatestRelease(slug string, branch string) (Release, error) {
+	var releases []Release
+	var release Release
+
+	resp, err := ghDefaultClient.getReleases(slug)
+	if err != nil {
+		return release, err
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return release, err
+	}
+
+	err = json.Unmarshal(body, &releases)
+	if err != nil {
+		return release, err
+	}
+
+	if branch != "" {
+		for _, r := range releases {
+			if r.TargetCommitish == branch {
+				return r, nil
+			}
+		}
+		return release, fmt.Errorf("failed to find release for branch %v", branch)
+	}
+
+	return releases[0], nil
+}
+
+func installRelease(repo Repo, release Release, regex string, pack core.Pack) error {
+	expr := regexp2.MustCompile(regex, 0)
+
+	if len(release.Assets) == 0 {
+		return errors.New("release doesn't have any assets attached")
+	}
+
+	var files []Asset
+
+	for _, v := range release.Assets {
+		bl, _ := expr.MatchString(v.Name)
+		if bl {
+			files = append(files, v)
+		}
+	}
+
+	if len(files) == 0 {
+		return errors.New("release doesn't have any assets matching regex")
+	}
+
+	if len(files) > 1 {
+		// TODO: also print file names
+		return errors.New("release has more than one asset matching regex")
+	}
+
+	file := files[0]
+
+	// Install the file
+	fmt.Printf("Installing %s from release %s\n", file.Name, release.TagName)
+	index, err := pack.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	updateMap := make(map[string]map[string]interface{})
+
+	updateMap["github"], err = ghUpdateData{
+		Slug:   repo.FullName,
+		Tag:    release.TagName,
+		Branch: release.TargetCommitish, // TODO: if no branch is specified by the user, we shouldn't record it - in order to remain branch-agnostic in getLatestRelease()
+		Regex:  regex,                   // TODO: ditto!
+	}.ToMap()
+	if err != nil {
+		return err
+	}
+
+	hash, err := file.getSha256()
+	if err != nil {
+		return err
+	}
+
+	modMeta := core.Mod{
+		Name:     repo.Name,
+		FileName: file.Name,
+		Side:     core.UniversalSide,
+		Download: core.ModDownload{
+			URL:        file.BrowserDownloadURL,
+			HashFormat: "sha256",
+			Hash:       hash,
+		},
+		Update: updateMap,
+	}
+	var path string
+	folder := viper.GetString("meta-folder")
+	if folder == "" {
+		folder = "mods"
+	}
+	path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, core.SlugifyName(repo.Name)+core.MetaExtension))
+
+	// If the file already exists, this will overwrite it!!!
+	// TODO: Should this be improved?
+	// Current strategy is to go ahead and do stuff without asking, with the assumption that you are using
+	// VCS anyway.
+
+	format, hash, err := modMeta.Write()
+	if err != nil {
+		return err
+	}
+
+	err = index.RefreshFileWithHash(path, format, hash, true)
+	if err != nil {
+		return err
+	}
+	err = index.Write()
+	if err != nil {
+		return err
+	}
+	err = pack.UpdateIndexHash()
+	if err != nil {
+		return err
+	}
+	err = pack.Write()
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Project \"%s\" successfully added! (%s)\n", repo.Name, file.Name)
+	return nil
+}
+
+var branchFlag string
+var regexFlag string
+
+func init() {
+	githubCmd.AddCommand(installCmd)
+
+	installCmd.Flags().StringVar(&branchFlag, "branch", "", "The GitHub repository branch to retrieve releases for")
+	installCmd.Flags().StringVar(&regexFlag, "regex", "", "The regular expression to match releases against")
+}

github/request.go

@@ -0,0 +1,81 @@
+package github
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/packwiz/packwiz/core"
+	"github.com/spf13/viper"
+)
+
+const ghApiServer = "api.github.com"
+
+type ghApiClient struct {
+	httpClient *http.Client
+}
+
+var ghDefaultClient = ghApiClient{&http.Client{}}
+
+func (c *ghApiClient) makeGet(url string) (*http.Response, error) {
+	ghApiToken := viper.GetString("github.token")
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("User-Agent", core.UserAgent)
+	req.Header.Set("Accept", "application/vnd.github+json")
+	if ghApiToken != "" {
+		req.Header.Set("Authorization", "Bearer "+ghApiToken)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: there is likely a better way to do this
+	ratelimit := 999
+
+	ratelimit_header := resp.Header.Get("x-ratelimit-remaining")
+	if ratelimit_header != "" {
+		ratelimit, err = strconv.Atoi(ratelimit_header)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if resp.StatusCode == 403 && ratelimit == 0 {
+		return nil, fmt.Errorf("GitHub API ratelimit exceeded; time of reset: %v", resp.Header.Get("x-ratelimit-reset"))
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid response status: %v", resp.Status)
+	}
+
+	if ratelimit < 10 {
+		fmt.Printf("Warning: GitHub API allows %v more requests before ratelimiting\n", ratelimit)
+		fmt.Println("Specifying a token is recommended; see documentation")
+	}
+
+	return resp, nil
+}
+
+func (c *ghApiClient) getRepo(slug string) (*http.Response, error) {
+	resp, err := c.makeGet("https://" + ghApiServer + "/repos/" + slug)
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}
+
+func (c *ghApiClient) getReleases(slug string) (*http.Response, error) {
+	resp, err := c.getRepo(slug + "/releases")
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}

github/updater.go

@@ -0,0 +1,123 @@
+package github
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/dlclark/regexp2"
+	"github.com/mitchellh/mapstructure"
+	"github.com/packwiz/packwiz/core"
+)
+
+type ghUpdateData struct {
+	Slug   string `mapstructure:"slug"`
+	Tag    string `mapstructure:"tag"`
+	Branch string `mapstructure:"branch"`
+	Regex  string `mapstructure:"regex"`
+}
+
+type ghUpdater struct{}
+
+func (u ghUpdater) ParseUpdate(updateUnparsed map[string]interface{}) (interface{}, error) {
+	var updateData ghUpdateData
+	err := mapstructure.Decode(updateUnparsed, &updateData)
+	return updateData, err
+}
+
+type cachedStateStore struct {
+	Slug    string
+	Release Release
+}
+
+func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateCheck, error) {
+	results := make([]core.UpdateCheck, len(mods))
+
+	for i, mod := range mods {
+		rawData, ok := mod.GetParsedUpdateData("github")
+		if !ok {
+			results[i] = core.UpdateCheck{Error: errors.New("failed to parse update metadata")}
+			continue
+		}
+
+		data := rawData.(ghUpdateData)
+
+		newRelease, err := getLatestRelease(data.Slug, data.Branch)
+		if err != nil {
+			results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest release: %v", err)}
+			continue
+		}
+
+		if newRelease.TagName == data.Tag { // The latest release is the same as the installed one
+			results[i] = core.UpdateCheck{UpdateAvailable: false}
+			continue
+		}
+
+		expr := regexp2.MustCompile(data.Regex, 0)
+
+		if len(newRelease.Assets) == 0 {
+			results[i] = core.UpdateCheck{Error: errors.New("new release doesn't have any assets")}
+			continue
+		}
+
+		var newFiles []Asset
+
+		for _, v := range newRelease.Assets {
+			bl, _ := expr.MatchString(v.Name)
+			if bl {
+				newFiles = append(newFiles, v)
+			}
+		}
+
+		if len(newFiles) == 0 {
+			results[i] = core.UpdateCheck{Error: errors.New("release doesn't have any assets matching regex")}
+			continue
+		}
+
+		if len(newFiles) > 1 {
+			// TODO: also print file names
+			results[i] = core.UpdateCheck{Error: errors.New("release has more than one asset matching regex")}
+			continue
+		}
+
+		newFile := newFiles[0]
+
+		results[i] = core.UpdateCheck{
+			UpdateAvailable: true,
+			UpdateString:    mod.FileName + " -> " + newFile.Name,
+			CachedState:     cachedStateStore{data.Slug, newRelease},
+		}
+	}
+
+	return results, nil
+}
+
+func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error {
+	for i, mod := range mods {
+		modState := cachedState[i].(cachedStateStore)
+		var release = modState.Release
+
+		// yes, this is duplicated - i guess we should just cache asset + tag instead of entire release...?
+		var file = release.Assets[0]
+		for _, v := range release.Assets {
+			if strings.HasSuffix(v.Name, ".jar") {
+				file = v
+			}
+		}
+
+		hash, err := file.getSha256()
+		if err != nil {
+			return err
+		}
+
+		mod.FileName = file.Name
+		mod.Download = core.ModDownload{
+			URL:        file.BrowserDownloadURL,
+			HashFormat: "sha256",
+			Hash:       hash,
+		}
+		mod.Update["github"]["tag"] = release.TagName
+	}
+
+	return nil
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aviddiviner/go-murmur v0.0.0-20150519214947-b9740d71e571
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect
+	github.com/dlclark/regexp2 v1.11.0
 	github.com/fatih/camelcase v1.0.0
 	github.com/igorsobreira/titlecase v0.0.0-20140109233139-4156b5b858ac
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -66,6 +66,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE=
 github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=

main.go

@@ -4,6 +4,7 @@ import (
 	// Modules of packwiz
 	"github.com/packwiz/packwiz/cmd"
 	_ "github.com/packwiz/packwiz/curseforge"
+	_ "github.com/packwiz/packwiz/github"
 	_ "github.com/packwiz/packwiz/migrate"
 	_ "github.com/packwiz/packwiz/modrinth"
 	_ "github.com/packwiz/packwiz/settings"