From 07033023afdee9e9809629a051a79e4e3633651c Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:14:03 +0200 Subject: [PATCH 01/22] feat: begin github support --- core/download.go | 3 +- github/github.go | 70 +++++++ github/install.go | 504 ++++++++++++++++++++++++++++++++++++++++++++++ github/updater.go | 102 ++++++++++ main.go | 1 + 5 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 github/github.go create mode 100644 github/install.go create mode 100644 github/updater.go diff --git a/core/download.go b/core/download.go index 066d956..4fe6558 100644 --- a/core/download.go +++ b/core/download.go @@ -4,12 +4,13 @@ import ( "encoding/json" "errors" "fmt" - "golang.org/x/exp/slices" "io" "net/http" "os" "path/filepath" "strings" + + "golang.org/x/exp/slices" ) const UserAgent = "packwiz/packwiz" diff --git a/github/github.go b/github/github.go new file mode 100644 index 0000000..b63ccdb --- /dev/null +++ b/github/github.go @@ -0,0 +1,70 @@ +package github + +import ( + "github.com/packwiz/packwiz/cmd" + "github.com/packwiz/packwiz/core" + "github.com/spf13/cobra" +) + +var githubCmd = &cobra.Command{ + Use: "github", + Aliases: []string{"gh"}, + Short: "Manage github-based mods", +} + +func init() { + cmd.Add(githubCmd) + core.Updaters["github"] = ghUpdater{} +} + +type ModReleases struct { + URL string `json:"url"` + AssetsURL string `json:"assets_url"` + UploadURL string `json:"upload_url"` + HTMLURL string `json:"html_url"` + ID int `json:"id"` + Author struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"author"` + NodeID string `json:"node_id"` + TagName string `json:"tag_name"` + TargetCommitish string `json:"target_commitish"` + Name string `json:"name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + CreatedAt string `json:"created_at"` + PublishedAt string `json:"published_at"` + Assets []Asset `json:"assets"` + TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url"` + Body string `json:"body"` + Reactions struct { + URL string `json:"url"` + TotalCount int `json:"total_count"` + Num1 int `json:"+1"` + Num10 int `json:"-1"` + Laugh int `json:"laugh"` + Hooray int `json:"hooray"` + Confused int `json:"confused"` + Heart int `json:"heart"` + Rocket int `json:"rocket"` + Eyes int `json:"eyes"` + } `json:"reactions"` +} diff --git a/github/install.go b/github/install.go new file mode 100644 index 0000000..a39941c --- /dev/null +++ b/github/install.go @@ -0,0 +1,504 @@ +package github + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/packwiz/packwiz/core" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var modSiteRegex = regexp.MustCompile("modrinth\\.com/mod/([^/]+)/?$") +var versionSiteRegex = regexp.MustCompile("modrinth\\.com/mod/([^/]+)/version/([^/]+)/?$") + +// installCmd represents the install command +var installCmd = &cobra.Command{ + Use: "install [mod]", + Short: "Install a mod from a github URL", + Aliases: []string{"add", "get"}, + Args: cobra.ArbitraryArgs, + Run: func(cmd *cobra.Command, args []string) { + pack, err := core.LoadPack() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if len(args) == 0 || len(args[0]) == 0 { + fmt.Println("You must specify a mod.") + os.Exit(1) + } + if strings.HasSuffix(args[0], "/") { + fmt.Println("Url cant have a leading slash!") + os.Exit(1) + } + + slug := strings.Replace(args[0], "https://github.com/", "", 1) + + mod, err := fetchMod(slug) + + installMod(mod, pack) + }, +} + +func init() { + githubCmd.AddCommand(installCmd) +} + +const githubApiUrl = "https://api.github.com/" + +func fetchMod(slug string) (Mod, error) { + var modReleases []ModReleases + var mod Mod + resp, err := http.Get(githubApiUrl + "repos/" + slug + "/releases") + if err != nil { + return mod, err + } + + if resp.StatusCode == 404 { + return mod, fmt.Errorf("mod not found (for URL %v)", githubApiUrl+"repos/"+slug+"/releases") + } + + if resp.StatusCode != 200 { + return mod, fmt.Errorf("invalid response status %v for URL %v", resp.Status, githubApiUrl+"repos/"+slug+"/releases") + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return mod, err + } + + err = json.Unmarshal(body, &modReleases) + if err != nil { + return mod, err + } + + var repoData Repo + + repoResp, err := http.Get(githubApiUrl + "repos/" + slug) + + defer repoResp.Body.Close() + repoBody, err := ioutil.ReadAll(repoResp.Body) + if err != nil { + return mod, err + } + + err = json.Unmarshal(repoBody, &repoData) + if err != nil { + return mod, err + } + + release := modReleases[0] + + mod = Mod{ + ID: repoData.Name, + Slug: slug, + Team: repoData.Owner.Login, + Title: repoData.Name, + Description: repoData.Description, + Published: repoData.CreatedAt, + Updated: release.CreatedAt, + License: repoData.License, + ClientSide: "unknown", + ServerSide: "unknown", + Categories: repoData.Topics, + } + if mod.ID == "" { + return mod, errors.New("invalid json whilst fetching mod: " + slug) + } + + return mod, nil + +} + +func installMod(mod Mod, pack core.Pack) error { + fmt.Printf("Found mod %s: '%s'.\n", mod.Title, mod.Description) + + latestVersion, err := getLatestVersion(mod.Slug, pack) + if err != nil { + return fmt.Errorf("failed to get latest version: %v", err) + } + if latestVersion.URL == "" { + return errors.New("mod is not available for this Minecraft version (use the acceptable-game-versions option to accept more) or mod loader") + } + + return installVersion(mod, latestVersion, pack) +} + +func getLatestVersion(slug string, pack core.Pack) (ModReleases, error) { + var modReleases []ModReleases + var release ModReleases + + resp, err := http.Get(githubApiUrl + "repos/" + slug + "/releases") + if err != nil { + return release, err + } + + if resp.StatusCode == 404 { + return release, fmt.Errorf("mod not found (for URL %v)", githubApiUrl+"repos/"+slug+"/releases") + } + + if resp.StatusCode != 200 { + return release, fmt.Errorf("invalid response status %v for URL %v", resp.Status, githubApiUrl+"repos/"+slug+"/releases") + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return release, err + } + err = json.Unmarshal(body, &modReleases) + if err != nil { + return release, err + } + + return modReleases[0], nil +} + +func installVersion(mod Mod, version ModReleases, pack core.Pack) error { + var files = version.Assets + + if len(files) == 0 { + return errors.New("version doesn't have any files attached") + } + + // TODO: add some way to allow users to pick which file to install? + var file = files[0] + for _, v := range version.Assets { + if strings.HasSuffix(v.Name, ".jar") { + file = v + } + } + + //Install the file + fmt.Printf("Installing %s from version %s\n", file.URL, version.Name) + index, err := pack.LoadIndex() + if err != nil { + return err + } + + updateMap := make(map[string]map[string]interface{}) + + updateMap["github"], err = ghUpdateData{ + ModID: mod.ID, + InstalledVersion: version.TagName, + }.ToMap() + if err != nil { + return err + } + + // side := mod.getSide() + // if side == "" { + // return errors.New("version doesn't have a side that's supported. Server: " + mod.ServerSide + " Client: " + mod.ClientSide) + // } + + hash, error := file.getSha256() + if error != nil || hash == "" { + return errors.New("file doesn't have a hash") + } + + modMeta := core.Mod{ + Name: mod.Title, + FileName: file.Name, + Side: "unknown", + Download: core.ModDownload{ + URL: file.BrowserDownloadURL, + HashFormat: "sha256", + Hash: hash, + }, + Update: updateMap, + } + var path string + folder := viper.GetString("meta-folder") + if folder == "" { + folder = "mods" + } + if mod.Slug != "" { + path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Slug+core.MetaExtension)) + } else { + path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Title+core.MetaExtension)) + } + + // If the file already exists, this will overwrite it!!! + // TODO: Should this be improved? + // Current strategy is to go ahead and do stuff without asking, with the assumption that you are using + // VCS anyway. + + format, hash, err := modMeta.Write() + if err != nil { + return err + } + err = index.RefreshFileWithHash(path, format, hash, true) + if err != nil { + return err + } + err = index.Write() + if err != nil { + return err + } + err = pack.UpdateIndexHash() + if err != nil { + return err + } + err = pack.Write() + if err != nil { + return err + } + return nil +} + +func (u ghUpdateData) ToMap() (map[string]interface{}, error) { + newMap := make(map[string]interface{}) + err := mapstructure.Decode(u, &newMap) + return newMap, err +} + +type License struct { + Id string `json:"id"` //The license id of a mod, retrieved from the licenses get route + Name string `json:"name"` //The long for name of a license + Url string `json:"url"` //The URL to this license +} + +type Mod struct { + ID string `json:"id"` //The ID of the mod, encoded as a base62 string + Slug string `json:"slug"` //The slug of a mod, used for vanity URLs + Team string `json:"team"` //The id of the team that has ownership of this mod + Title string `json:"title"` //The title or name of the mod + Description string `json:"description"` //A short description of the mod + // Body string `json:"body"` //A long form description of the mod. + // BodyUrl string `json:"body_url"` //DEPRECATED The link to the long description of the mod (Optional) + Published string `json:"published"` //The date at which the mod was first published + Updated string `json:"updated"` //The date at which the mod was updated + License struct { + Key string `json:"key"` + Name string `json:"name"` + SpdxID string `json:"spdx_id"` + URL string `json:"url"` + NodeID string `json:"node_id"` + } `json:"license"` + ClientSide string `json:"client_side"` //The support range for the client mod - required, optional, unsupported, or unknown + ServerSide string `json:"server_side"` //The support range for the server mod - required, optional, unsupported, or unknown + // Downloads int `json:"downloads"` //The total number of downloads the mod has + Categories []string `json:"categories"` //A list of the categories that the mod is in + Versions []string `json:"versions"` //A list of ids for versions of the mod + IconUrl string `json:"icon_url"` //The URL of the icon of the mod (Optional) + IssuesUrl string `json:"issues_url"` //An optional link to where to submit bugs or issues with the mod (Optional) + SourceUrl string `json:"source_url"` //An optional link to the source code for the mod (Optional) + WikiUrl string `json:"wiki_url"` //An optional link to the mod's wiki page or other relevant information (Optional) + DiscordUrl string `json:"discord_url"` //An optional link to the mod's discord (Optional) +} + +type Asset struct { + URL string `json:"url"` + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + Label interface{} `json:"label"` + Uploader struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"uploader"` + ContentType string `json:"content_type"` + State string `json:"state"` + Size int `json:"size"` + DownloadCount int `json:"download_count"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +func (u Asset) getSha256() (string, error) { + // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! + file, err := os.CreateTemp("", "download") + + if err != nil { + fmt.Println(err) + os.Exit(1) + } + mainHasher, err := core.GetHashImpl("sha256") + resp, err := http.Get(u.BrowserDownloadURL) + if err != nil { + return "", err + } + if resp.StatusCode == 404 { + return "", fmt.Errorf("Asset not found") + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("Invalid response code: %d", resp.StatusCode) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + file.Write(body) + + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + log.Fatal(err) + } + + hash := h.Sum(nil) + + defer os.Remove(file.Name()) + + return mainHasher.HashToString(hash), nil +} + +type Repo struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"owner"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + URL string `json:"url"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + DeploymentsURL string `json:"deployments_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + PushedAt string `json:"pushed_at"` + GitURL string `json:"git_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + SvnURL string `json:"svn_url"` + Homepage string `json:"homepage"` + Size int `json:"size"` + StargazersCount int `json:"stargazers_count"` + WatchersCount int `json:"watchers_count"` + Language string `json:"language"` + HasIssues bool `json:"has_issues"` + HasProjects bool `json:"has_projects"` + HasDownloads bool `json:"has_downloads"` + HasWiki bool `json:"has_wiki"` + HasPages bool `json:"has_pages"` + ForksCount int `json:"forks_count"` + MirrorURL interface{} `json:"mirror_url"` + Archived bool `json:"archived"` + Disabled bool `json:"disabled"` + OpenIssuesCount int `json:"open_issues_count"` + License struct { + Key string `json:"key"` + Name string `json:"name"` + SpdxID string `json:"spdx_id"` + URL string `json:"url"` + NodeID string `json:"node_id"` + } `json:"license"` + AllowForking bool `json:"allow_forking"` + IsTemplate bool `json:"is_template"` + Topics []string `json:"topics"` + Visibility string `json:"visibility"` + Forks int `json:"forks"` + OpenIssues int `json:"open_issues"` + Watchers int `json:"watchers"` + DefaultBranch string `json:"default_branch"` + TempCloneToken interface{} `json:"temp_clone_token"` + Organization struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"organization"` + NetworkCount int `json:"network_count"` + SubscribersCount int `json:"subscribers_count"` +} diff --git a/github/updater.go b/github/updater.go new file mode 100644 index 0000000..647a2ba --- /dev/null +++ b/github/updater.go @@ -0,0 +1,102 @@ +package github + +import ( + "errors" + "fmt" + "strings" + + "github.com/mitchellh/mapstructure" + "github.com/packwiz/packwiz/core" +) + +type ghUpdateData struct { + ModID string `mapstructure:"mod-id"` // The slug of the repo but named modId for consistency reasons + InstalledVersion string `mapstructure:"version"` +} + +type ghUpdater struct{} + +func (u ghUpdater) ParseUpdate(updateUnparsed map[string]interface{}) (interface{}, error) { + var updateData ghUpdateData + err := mapstructure.Decode(updateUnparsed, &updateData) + return updateData, err +} + +type cachedStateStore struct { + ModID string + Version ModReleases +} + +func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack) ([]core.UpdateCheck, error) { + results := make([]core.UpdateCheck, len(mods)) + + for i, mod := range mods { + rawData, ok := mod.GetParsedUpdateData("modrinth") + if !ok { + results[i] = core.UpdateCheck{Error: errors.New("couldn't parse mod data")} + continue + } + + data := rawData.(ghUpdateData) + + newVersion, err := getLatestVersion(data.ModID, pack) + if err != nil { + results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest version: %v", err)} + continue + } + + // if newVersion.ID == "" { //There is no version available for this minecraft version or loader. + // results[i] = core.UpdateCheck{UpdateAvailable: false} + // continue + // } + + if newVersion.TagName == data.InstalledVersion { //The latest version from the site is the same as the installed one + results[i] = core.UpdateCheck{UpdateAvailable: false} + continue + } + + if len(newVersion.Assets) == 0 { + results[i] = core.UpdateCheck{Error: errors.New("new version doesn't have any files")} + continue + } + + newFilename := newVersion.Assets[0].Name + + results[i] = core.UpdateCheck{ + UpdateAvailable: true, + UpdateString: mod.FileName + " -> " + newFilename, + CachedState: cachedStateStore{data.ModID, newVersion}, + } + } + + return results, nil +} + +func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { + for i, mod := range mods { + modState := cachedState[i].(cachedStateStore) + var version = modState.Version + + var file = version.Assets[0] + for _, v := range version.Assets { + if strings.HasSuffix(v.Name, ".jar") { + file = v + } + } + + hash, error := file.getSha256() + if error != nil || hash == "" { + return errors.New("file doesn't have a hash") + } + + mod.FileName = file.Name + mod.Download = core.ModDownload{ + URL: file.BrowserDownloadURL, + HashFormat: "sha256", + Hash: hash, + } + mod.Update["modrinth"]["version"] = version.ID + } + + return nil +} diff --git a/main.go b/main.go index 82e35b8..d5c3a42 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( // Modules of packwiz "github.com/packwiz/packwiz/cmd" _ "github.com/packwiz/packwiz/curseforge" + _ "github.com/packwiz/packwiz/github" _ "github.com/packwiz/packwiz/migrate" _ "github.com/packwiz/packwiz/modrinth" _ "github.com/packwiz/packwiz/settings" From 715e9439a1fcd0edf33dc1bda125c0c386c8443b Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:26:32 +0200 Subject: [PATCH 02/22] fix: file hashing --- github/install.go | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/github/install.go b/github/install.go index a39941c..034ed81 100644 --- a/github/install.go +++ b/github/install.go @@ -1,13 +1,10 @@ package github import ( - "crypto/sha256" "encoding/json" "errors" "fmt" - "io" "io/ioutil" - "log" "net/http" "os" "path/filepath" @@ -339,12 +336,6 @@ type Asset struct { func (u Asset) getSha256() (string, error) { // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! - file, err := os.CreateTemp("", "download") - - if err != nil { - fmt.Println(err) - os.Exit(1) - } mainHasher, err := core.GetHashImpl("sha256") resp, err := http.Get(u.BrowserDownloadURL) if err != nil { @@ -360,17 +351,9 @@ func (u Asset) getSha256() (string, error) { defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) + mainHasher.Write(body) - file.Write(body) - - h := sha256.New() - if _, err := io.Copy(h, file); err != nil { - log.Fatal(err) - } - - hash := h.Sum(nil) - - defer os.Remove(file.Name()) + hash := mainHasher.Sum(nil) return mainHasher.HashToString(hash), nil } From 8c97b3e73b3a3b43e3a49f502137a911bfffba96 Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:36:18 +0200 Subject: [PATCH 03/22] fix: use mod.title instead of slug --- github/install.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/github/install.go b/github/install.go index 034ed81..b5c662f 100644 --- a/github/install.go +++ b/github/install.go @@ -224,11 +224,7 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { if folder == "" { folder = "mods" } - if mod.Slug != "" { - path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Slug+core.MetaExtension)) - } else { - path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Title+core.MetaExtension)) - } + path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Title+core.MetaExtension)) // If the file already exists, this will overwrite it!!! // TODO: Should this be improved? From cb9550f4a61454fe9178a095a4726dd46816c2d2 Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:15:31 +0200 Subject: [PATCH 04/22] feat: add support for more url aliases --- github/install.go | 14 +++++++++----- github/updater.go | 5 ----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/github/install.go b/github/install.go index b5c662f..1ac9113 100644 --- a/github/install.go +++ b/github/install.go @@ -29,6 +29,7 @@ var installCmd = &cobra.Command{ Args: cobra.ArbitraryArgs, Run: func(cmd *cobra.Command, args []string) { pack, err := core.LoadPack() + if err != nil { fmt.Println(err) os.Exit(1) @@ -45,6 +46,14 @@ var installCmd = &cobra.Command{ slug := strings.Replace(args[0], "https://github.com/", "", 1) + if len(strings.Split(args[0], "/")) == 1 { + slug = args[0] + } + + if strings.Contains(slug, "/releases") { + slug = strings.Split(slug, "/releases")[0] + } + mod, err := fetchMod(slug) installMod(mod, pack) @@ -198,11 +207,6 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { return err } - // side := mod.getSide() - // if side == "" { - // return errors.New("version doesn't have a side that's supported. Server: " + mod.ServerSide + " Client: " + mod.ClientSide) - // } - hash, error := file.getSha256() if error != nil || hash == "" { return errors.New("file doesn't have a hash") diff --git a/github/updater.go b/github/updater.go index 647a2ba..138b963 100644 --- a/github/updater.go +++ b/github/updater.go @@ -45,11 +45,6 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack continue } - // if newVersion.ID == "" { //There is no version available for this minecraft version or loader. - // results[i] = core.UpdateCheck{UpdateAvailable: false} - // continue - // } - if newVersion.TagName == data.InstalledVersion { //The latest version from the site is the same as the installed one results[i] = core.UpdateCheck{UpdateAvailable: false} continue From 5c07741447c8674529697060a1ffb6a4ccdcf5c6 Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:38:07 +0200 Subject: [PATCH 05/22] feat: add branch support for github mods --- github/github.go | 2 +- github/install.go | 16 +++++++++------- github/updater.go | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/github/github.go b/github/github.go index b63ccdb..a61c2ad 100644 --- a/github/github.go +++ b/github/github.go @@ -45,7 +45,7 @@ type ModReleases struct { } `json:"author"` NodeID string `json:"node_id"` TagName string `json:"tag_name"` - TargetCommitish string `json:"target_commitish"` + TargetCommitish string `json:"target_commitish"` // The branch of the release Name string `json:"name"` Draft bool `json:"draft"` Prerelease bool `json:"prerelease"` diff --git a/github/install.go b/github/install.go index 1ac9113..9a7aa2d 100644 --- a/github/install.go +++ b/github/install.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" "time" @@ -18,9 +17,6 @@ import ( "github.com/spf13/viper" ) -var modSiteRegex = regexp.MustCompile("modrinth\\.com/mod/([^/]+)/?$") -var versionSiteRegex = regexp.MustCompile("modrinth\\.com/mod/([^/]+)/version/([^/]+)/?$") - // installCmd represents the install command var installCmd = &cobra.Command{ Use: "install [mod]", @@ -134,7 +130,7 @@ func fetchMod(slug string) (Mod, error) { func installMod(mod Mod, pack core.Pack) error { fmt.Printf("Found mod %s: '%s'.\n", mod.Title, mod.Description) - latestVersion, err := getLatestVersion(mod.Slug, pack) + latestVersion, err := getLatestVersion(mod.Slug, pack, "") if err != nil { return fmt.Errorf("failed to get latest version: %v", err) } @@ -145,7 +141,7 @@ func installMod(mod Mod, pack core.Pack) error { return installVersion(mod, latestVersion, pack) } -func getLatestVersion(slug string, pack core.Pack) (ModReleases, error) { +func getLatestVersion(slug string, pack core.Pack, branch string) (ModReleases, error) { var modReleases []ModReleases var release ModReleases @@ -171,6 +167,11 @@ func getLatestVersion(slug string, pack core.Pack) (ModReleases, error) { if err != nil { return release, err } + for _, r := range modReleases { + if r.TargetCommitish == branch { + return r, nil + } + } return modReleases[0], nil } @@ -200,8 +201,9 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { updateMap := make(map[string]map[string]interface{}) updateMap["github"], err = ghUpdateData{ - ModID: mod.ID, + ModID: mod.Slug, InstalledVersion: version.TagName, + Branch: version.TargetCommitish, }.ToMap() if err != nil { return err diff --git a/github/updater.go b/github/updater.go index 138b963..587d864 100644 --- a/github/updater.go +++ b/github/updater.go @@ -12,6 +12,7 @@ import ( type ghUpdateData struct { ModID string `mapstructure:"mod-id"` // The slug of the repo but named modId for consistency reasons InstalledVersion string `mapstructure:"version"` + Branch string `mapstructure:"branch"` } type ghUpdater struct{} @@ -31,7 +32,7 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack results := make([]core.UpdateCheck, len(mods)) for i, mod := range mods { - rawData, ok := mod.GetParsedUpdateData("modrinth") + rawData, ok := mod.GetParsedUpdateData("github") if !ok { results[i] = core.UpdateCheck{Error: errors.New("couldn't parse mod data")} continue @@ -39,7 +40,7 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack data := rawData.(ghUpdateData) - newVersion, err := getLatestVersion(data.ModID, pack) + newVersion, err := getLatestVersion(data.ModID, pack, data.Branch) if err != nil { results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest version: %v", err)} continue @@ -90,7 +91,7 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { HashFormat: "sha256", Hash: hash, } - mod.Update["modrinth"]["version"] = version.ID + mod.Update["github"]["version"] = version.ID } return nil From faec4f47389055f8fdfa15514e154a0cb81553da Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:17:55 +0200 Subject: [PATCH 06/22] use sha1 instead of sha256 --- github/install.go | 13 +++++++++---- github/updater.go | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/github/install.go b/github/install.go index 9a7aa2d..7a9437b 100644 --- a/github/install.go +++ b/github/install.go @@ -93,7 +93,12 @@ func fetchMod(slug string) (Mod, error) { repoResp, err := http.Get(githubApiUrl + "repos/" + slug) + if err != nil { + return mod, err + } + defer repoResp.Body.Close() + repoBody, err := ioutil.ReadAll(repoResp.Body) if err != nil { return mod, err @@ -209,7 +214,7 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { return err } - hash, error := file.getSha256() + hash, error := file.getSha1() if error != nil || hash == "" { return errors.New("file doesn't have a hash") } @@ -220,7 +225,7 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { Side: "unknown", Download: core.ModDownload{ URL: file.BrowserDownloadURL, - HashFormat: "sha256", + HashFormat: "sha1", Hash: hash, }, Update: updateMap, @@ -336,9 +341,9 @@ type Asset struct { BrowserDownloadURL string `json:"browser_download_url"` } -func (u Asset) getSha256() (string, error) { +func (u Asset) getSha1() (string, error) { // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! - mainHasher, err := core.GetHashImpl("sha256") + mainHasher, err := core.GetHashImpl("sha1") resp, err := http.Get(u.BrowserDownloadURL) if err != nil { return "", err diff --git a/github/updater.go b/github/updater.go index 587d864..32e74a8 100644 --- a/github/updater.go +++ b/github/updater.go @@ -80,7 +80,7 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { } } - hash, error := file.getSha256() + hash, error := file.getSha1() if error != nil || hash == "" { return errors.New("file doesn't have a hash") } @@ -88,7 +88,7 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { mod.FileName = file.Name mod.Download = core.ModDownload{ URL: file.BrowserDownloadURL, - HashFormat: "sha256", + HashFormat: "sha1", Hash: hash, } mod.Update["github"]["version"] = version.ID From 837b4db76062e1cad51d4a3821de1e6f88ab6721 Mon Sep 17 00:00:00 2001 From: Tricked <72335827+SkyBlockDev@users.noreply.github.com> Date: Thu, 16 Jun 2022 07:51:06 +0200 Subject: [PATCH 07/22] fix: apply some suggestions --- github/github.go | 203 ++++++++++++++++++++++------ github/install.go | 337 +++------------------------------------------- github/updater.go | 8 +- 3 files changed, 186 insertions(+), 362 deletions(-) diff --git a/github/github.go b/github/github.go index a61c2ad..33881c0 100644 --- a/github/github.go +++ b/github/github.go @@ -1,6 +1,14 @@ package github import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "time" + + "github.com/mitchellh/mapstructure" "github.com/packwiz/packwiz/cmd" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" @@ -17,54 +25,163 @@ func init() { core.Updaters["github"] = ghUpdater{} } +func fetchRepo(slug string) (Repo, error) { + var repo Repo + res, err := http.Get(githubApiUrl + "repos/" + slug) + + if err != nil { + return repo, err + } + + defer res.Body.Close() + + repoBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return repo, err + } + + err = json.Unmarshal(repoBody, &repo) + if err != nil { + return repo, err + } + return repo, nil +} + +func fetchMod(slug string) (Mod, error) { + + var mod Mod + + repo, err := fetchRepo(slug) + + if err != nil { + return mod, err + } + + release, err := getLatestVersion(slug, "") + + if err != nil { + return mod, err + } + + mod = Mod{ + ID: repo.Name, + Slug: slug, + Team: repo.Owner.Login, + Title: repo.Name, + Description: repo.Description, + Published: repo.CreatedAt, + Updated: release.CreatedAt, + License: repo.License, + ClientSide: "unknown", + ServerSide: "unknown", + Categories: repo.Topics, + } + if mod.ID == "" { + return mod, errors.New("invalid json whilst fetching mod: " + slug) + } + + return mod, nil + +} + type ModReleases struct { - URL string `json:"url"` - AssetsURL string `json:"assets_url"` - UploadURL string `json:"upload_url"` - HTMLURL string `json:"html_url"` - ID int `json:"id"` - Author struct { - Login string `json:"login"` - ID int `json:"id"` - NodeID string `json:"node_id"` - AvatarURL string `json:"avatar_url"` - GravatarID string `json:"gravatar_id"` - URL string `json:"url"` - HTMLURL string `json:"html_url"` - FollowersURL string `json:"followers_url"` - FollowingURL string `json:"following_url"` - GistsURL string `json:"gists_url"` - StarredURL string `json:"starred_url"` - SubscriptionsURL string `json:"subscriptions_url"` - OrganizationsURL string `json:"organizations_url"` - ReposURL string `json:"repos_url"` - EventsURL string `json:"events_url"` - ReceivedEventsURL string `json:"received_events_url"` - Type string `json:"type"` - SiteAdmin bool `json:"site_admin"` - } `json:"author"` + URL string `json:"url"` NodeID string `json:"node_id"` TagName string `json:"tag_name"` TargetCommitish string `json:"target_commitish"` // The branch of the release Name string `json:"name"` - Draft bool `json:"draft"` - Prerelease bool `json:"prerelease"` CreatedAt string `json:"created_at"` - PublishedAt string `json:"published_at"` Assets []Asset `json:"assets"` - TarballURL string `json:"tarball_url"` - ZipballURL string `json:"zipball_url"` - Body string `json:"body"` - Reactions struct { - URL string `json:"url"` - TotalCount int `json:"total_count"` - Num1 int `json:"+1"` - Num10 int `json:"-1"` - Laugh int `json:"laugh"` - Hooray int `json:"hooray"` - Confused int `json:"confused"` - Heart int `json:"heart"` - Rocket int `json:"rocket"` - Eyes int `json:"eyes"` - } `json:"reactions"` +} + +type Asset struct { + URL string `json:"url"` + Name string `json:"name"` + UpdatedAt time.Time `json:"updated_at"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +type Repo struct { + ID int `json:"id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Owner struct { + Login string `json:"login"` + } `json:"owner"` + Description string `json:"description"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + License struct { + Key string `json:"key"` + Name string `json:"name"` + SpdxID string `json:"spdx_id"` + URL string `json:"url"` + NodeID string `json:"node_id"` + } `json:"license"` + Topics []string `json:"topics"` +} + +func (u ghUpdateData) ToMap() (map[string]interface{}, error) { + newMap := make(map[string]interface{}) + err := mapstructure.Decode(u, &newMap) + return newMap, err +} + +type License struct { + Id string `json:"id"` //The license id of a mod, retrieved from the licenses get route + Name string `json:"name"` //The long for name of a license + Url string `json:"url"` //The URL to this license +} + +type Mod struct { + ID string `json:"id"` //The ID of the mod, encoded as a base62 string + Slug string `json:"slug"` //The slug of a mod, used for vanity URLs + Team string `json:"team"` //The id of the team that has ownership of this mod + Title string `json:"title"` //The title or name of the mod + Description string `json:"description"` //A short description of the mod + // Body string `json:"body"` //A long form description of the mod. + // BodyUrl string `json:"body_url"` //DEPRECATED The link to the long description of the mod (Optional) + Published string `json:"published"` //The date at which the mod was first published + Updated string `json:"updated"` //The date at which the mod was updated + License struct { + Key string `json:"key"` + Name string `json:"name"` + SpdxID string `json:"spdx_id"` + URL string `json:"url"` + NodeID string `json:"node_id"` + } `json:"license"` + ClientSide string `json:"client_side"` //The support range for the client mod - required, optional, unsupported, or unknown + ServerSide string `json:"server_side"` //The support range for the server mod - required, optional, unsupported, or unknown + // Downloads int `json:"downloads"` //The total number of downloads the mod has + Categories []string `json:"categories"` //A list of the categories that the mod is in + Versions []string `json:"versions"` //A list of ids for versions of the mod + IconUrl string `json:"icon_url"` //The URL of the icon of the mod (Optional) + IssuesUrl string `json:"issues_url"` //An optional link to where to submit bugs or issues with the mod (Optional) + SourceUrl string `json:"source_url"` //An optional link to the source code for the mod (Optional) + WikiUrl string `json:"wiki_url"` //An optional link to the mod's wiki page or other relevant information (Optional) + DiscordUrl string `json:"discord_url"` //An optional link to the mod's discord (Optional) +} + +func (u Asset) getSha1() (string, error) { + // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! + mainHasher, err := core.GetHashImpl("sha1") + resp, err := http.Get(u.BrowserDownloadURL) + if err != nil { + return "", err + } + if resp.StatusCode == 404 { + return "", fmt.Errorf("Asset not found") + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("Invalid response code: %d", resp.StatusCode) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + mainHasher.Write(body) + + hash := mainHasher.Sum(nil) + + return mainHasher.HashToString(hash), nil } diff --git a/github/install.go b/github/install.go index 7a9437b..7765557 100644 --- a/github/install.go +++ b/github/install.go @@ -8,19 +8,20 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strings" - "time" - "github.com/mitchellh/mapstructure" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" "github.com/spf13/viper" ) +var GithubRegex = regexp.MustCompile("https?://(?:www\\.)?github\\.com/([^/]+/[^/]+)") + // installCmd represents the install command var installCmd = &cobra.Command{ Use: "install [mod]", - Short: "Install a mod from a github URL", + Short: "Install mods from github releases", Aliases: []string{"add", "get"}, Args: cobra.ArbitraryArgs, Run: func(cmd *cobra.Command, args []string) { @@ -35,23 +36,27 @@ var installCmd = &cobra.Command{ fmt.Println("You must specify a mod.") os.Exit(1) } - if strings.HasSuffix(args[0], "/") { - fmt.Println("Url cant have a leading slash!") - os.Exit(1) - } - slug := strings.Replace(args[0], "https://github.com/", "", 1) + //Try interpreting the arg as a modId or slug. + //Modrinth transparently handles slugs/mod ids in their api; we don't have to detect which one it is. + var slug string - if len(strings.Split(args[0], "/")) == 1 { + //Try to see if it's a site, if extract the id/slug from the url. + //Otherwise, interpret the arg as a id/slug straight up + matches := GithubRegex.FindStringSubmatch(args[0]) + if matches != nil && len(matches) == 2 { + slug = matches[1] + } else { slug = args[0] } - if strings.Contains(slug, "/releases") { - slug = strings.Split(slug, "/releases")[0] - } - mod, err := fetchMod(slug) + if err != nil { + fmt.Println("Failed to get the mod ", err) + os.Exit(1) + } + installMod(mod, pack) }, } @@ -62,80 +67,10 @@ func init() { const githubApiUrl = "https://api.github.com/" -func fetchMod(slug string) (Mod, error) { - var modReleases []ModReleases - var mod Mod - resp, err := http.Get(githubApiUrl + "repos/" + slug + "/releases") - if err != nil { - return mod, err - } - - if resp.StatusCode == 404 { - return mod, fmt.Errorf("mod not found (for URL %v)", githubApiUrl+"repos/"+slug+"/releases") - } - - if resp.StatusCode != 200 { - return mod, fmt.Errorf("invalid response status %v for URL %v", resp.Status, githubApiUrl+"repos/"+slug+"/releases") - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return mod, err - } - - err = json.Unmarshal(body, &modReleases) - if err != nil { - return mod, err - } - - var repoData Repo - - repoResp, err := http.Get(githubApiUrl + "repos/" + slug) - - if err != nil { - return mod, err - } - - defer repoResp.Body.Close() - - repoBody, err := ioutil.ReadAll(repoResp.Body) - if err != nil { - return mod, err - } - - err = json.Unmarshal(repoBody, &repoData) - if err != nil { - return mod, err - } - - release := modReleases[0] - - mod = Mod{ - ID: repoData.Name, - Slug: slug, - Team: repoData.Owner.Login, - Title: repoData.Name, - Description: repoData.Description, - Published: repoData.CreatedAt, - Updated: release.CreatedAt, - License: repoData.License, - ClientSide: "unknown", - ServerSide: "unknown", - Categories: repoData.Topics, - } - if mod.ID == "" { - return mod, errors.New("invalid json whilst fetching mod: " + slug) - } - - return mod, nil - -} - func installMod(mod Mod, pack core.Pack) error { - fmt.Printf("Found mod %s: '%s'.\n", mod.Title, mod.Description) + fmt.Printf("Found repo %s: '%s'.\n", mod.Slug, mod.Description) - latestVersion, err := getLatestVersion(mod.Slug, pack, "") + latestVersion, err := getLatestVersion(mod.Slug, "") if err != nil { return fmt.Errorf("failed to get latest version: %v", err) } @@ -146,7 +81,7 @@ func installMod(mod Mod, pack core.Pack) error { return installVersion(mod, latestVersion, pack) } -func getLatestVersion(slug string, pack core.Pack, branch string) (ModReleases, error) { +func getLatestVersion(slug string, branch string) (ModReleases, error) { var modReleases []ModReleases var release ModReleases @@ -206,7 +141,7 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { updateMap := make(map[string]map[string]interface{}) updateMap["github"], err = ghUpdateData{ - ModID: mod.Slug, + Slug: mod.Slug, InstalledVersion: version.TagName, Branch: version.TargetCommitish, }.ToMap() @@ -264,231 +199,3 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { } return nil } - -func (u ghUpdateData) ToMap() (map[string]interface{}, error) { - newMap := make(map[string]interface{}) - err := mapstructure.Decode(u, &newMap) - return newMap, err -} - -type License struct { - Id string `json:"id"` //The license id of a mod, retrieved from the licenses get route - Name string `json:"name"` //The long for name of a license - Url string `json:"url"` //The URL to this license -} - -type Mod struct { - ID string `json:"id"` //The ID of the mod, encoded as a base62 string - Slug string `json:"slug"` //The slug of a mod, used for vanity URLs - Team string `json:"team"` //The id of the team that has ownership of this mod - Title string `json:"title"` //The title or name of the mod - Description string `json:"description"` //A short description of the mod - // Body string `json:"body"` //A long form description of the mod. - // BodyUrl string `json:"body_url"` //DEPRECATED The link to the long description of the mod (Optional) - Published string `json:"published"` //The date at which the mod was first published - Updated string `json:"updated"` //The date at which the mod was updated - License struct { - Key string `json:"key"` - Name string `json:"name"` - SpdxID string `json:"spdx_id"` - URL string `json:"url"` - NodeID string `json:"node_id"` - } `json:"license"` - ClientSide string `json:"client_side"` //The support range for the client mod - required, optional, unsupported, or unknown - ServerSide string `json:"server_side"` //The support range for the server mod - required, optional, unsupported, or unknown - // Downloads int `json:"downloads"` //The total number of downloads the mod has - Categories []string `json:"categories"` //A list of the categories that the mod is in - Versions []string `json:"versions"` //A list of ids for versions of the mod - IconUrl string `json:"icon_url"` //The URL of the icon of the mod (Optional) - IssuesUrl string `json:"issues_url"` //An optional link to where to submit bugs or issues with the mod (Optional) - SourceUrl string `json:"source_url"` //An optional link to the source code for the mod (Optional) - WikiUrl string `json:"wiki_url"` //An optional link to the mod's wiki page or other relevant information (Optional) - DiscordUrl string `json:"discord_url"` //An optional link to the mod's discord (Optional) -} - -type Asset struct { - URL string `json:"url"` - ID int `json:"id"` - NodeID string `json:"node_id"` - Name string `json:"name"` - Label interface{} `json:"label"` - Uploader struct { - Login string `json:"login"` - ID int `json:"id"` - NodeID string `json:"node_id"` - AvatarURL string `json:"avatar_url"` - GravatarID string `json:"gravatar_id"` - URL string `json:"url"` - HTMLURL string `json:"html_url"` - FollowersURL string `json:"followers_url"` - FollowingURL string `json:"following_url"` - GistsURL string `json:"gists_url"` - StarredURL string `json:"starred_url"` - SubscriptionsURL string `json:"subscriptions_url"` - OrganizationsURL string `json:"organizations_url"` - ReposURL string `json:"repos_url"` - EventsURL string `json:"events_url"` - ReceivedEventsURL string `json:"received_events_url"` - Type string `json:"type"` - SiteAdmin bool `json:"site_admin"` - } `json:"uploader"` - ContentType string `json:"content_type"` - State string `json:"state"` - Size int `json:"size"` - DownloadCount int `json:"download_count"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - BrowserDownloadURL string `json:"browser_download_url"` -} - -func (u Asset) getSha1() (string, error) { - // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! - mainHasher, err := core.GetHashImpl("sha1") - resp, err := http.Get(u.BrowserDownloadURL) - if err != nil { - return "", err - } - if resp.StatusCode == 404 { - return "", fmt.Errorf("Asset not found") - } - - if resp.StatusCode != 200 { - return "", fmt.Errorf("Invalid response code: %d", resp.StatusCode) - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - mainHasher.Write(body) - - hash := mainHasher.Sum(nil) - - return mainHasher.HashToString(hash), nil -} - -type Repo struct { - ID int `json:"id"` - NodeID string `json:"node_id"` - Name string `json:"name"` - FullName string `json:"full_name"` - Private bool `json:"private"` - Owner struct { - Login string `json:"login"` - ID int `json:"id"` - NodeID string `json:"node_id"` - AvatarURL string `json:"avatar_url"` - GravatarID string `json:"gravatar_id"` - URL string `json:"url"` - HTMLURL string `json:"html_url"` - FollowersURL string `json:"followers_url"` - FollowingURL string `json:"following_url"` - GistsURL string `json:"gists_url"` - StarredURL string `json:"starred_url"` - SubscriptionsURL string `json:"subscriptions_url"` - OrganizationsURL string `json:"organizations_url"` - ReposURL string `json:"repos_url"` - EventsURL string `json:"events_url"` - ReceivedEventsURL string `json:"received_events_url"` - Type string `json:"type"` - SiteAdmin bool `json:"site_admin"` - } `json:"owner"` - HTMLURL string `json:"html_url"` - Description string `json:"description"` - Fork bool `json:"fork"` - URL string `json:"url"` - ForksURL string `json:"forks_url"` - KeysURL string `json:"keys_url"` - CollaboratorsURL string `json:"collaborators_url"` - TeamsURL string `json:"teams_url"` - HooksURL string `json:"hooks_url"` - IssueEventsURL string `json:"issue_events_url"` - EventsURL string `json:"events_url"` - AssigneesURL string `json:"assignees_url"` - BranchesURL string `json:"branches_url"` - TagsURL string `json:"tags_url"` - BlobsURL string `json:"blobs_url"` - GitTagsURL string `json:"git_tags_url"` - GitRefsURL string `json:"git_refs_url"` - TreesURL string `json:"trees_url"` - StatusesURL string `json:"statuses_url"` - LanguagesURL string `json:"languages_url"` - StargazersURL string `json:"stargazers_url"` - ContributorsURL string `json:"contributors_url"` - SubscribersURL string `json:"subscribers_url"` - SubscriptionURL string `json:"subscription_url"` - CommitsURL string `json:"commits_url"` - GitCommitsURL string `json:"git_commits_url"` - CommentsURL string `json:"comments_url"` - IssueCommentURL string `json:"issue_comment_url"` - ContentsURL string `json:"contents_url"` - CompareURL string `json:"compare_url"` - MergesURL string `json:"merges_url"` - ArchiveURL string `json:"archive_url"` - DownloadsURL string `json:"downloads_url"` - IssuesURL string `json:"issues_url"` - PullsURL string `json:"pulls_url"` - MilestonesURL string `json:"milestones_url"` - NotificationsURL string `json:"notifications_url"` - LabelsURL string `json:"labels_url"` - ReleasesURL string `json:"releases_url"` - DeploymentsURL string `json:"deployments_url"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - PushedAt string `json:"pushed_at"` - GitURL string `json:"git_url"` - SSHURL string `json:"ssh_url"` - CloneURL string `json:"clone_url"` - SvnURL string `json:"svn_url"` - Homepage string `json:"homepage"` - Size int `json:"size"` - StargazersCount int `json:"stargazers_count"` - WatchersCount int `json:"watchers_count"` - Language string `json:"language"` - HasIssues bool `json:"has_issues"` - HasProjects bool `json:"has_projects"` - HasDownloads bool `json:"has_downloads"` - HasWiki bool `json:"has_wiki"` - HasPages bool `json:"has_pages"` - ForksCount int `json:"forks_count"` - MirrorURL interface{} `json:"mirror_url"` - Archived bool `json:"archived"` - Disabled bool `json:"disabled"` - OpenIssuesCount int `json:"open_issues_count"` - License struct { - Key string `json:"key"` - Name string `json:"name"` - SpdxID string `json:"spdx_id"` - URL string `json:"url"` - NodeID string `json:"node_id"` - } `json:"license"` - AllowForking bool `json:"allow_forking"` - IsTemplate bool `json:"is_template"` - Topics []string `json:"topics"` - Visibility string `json:"visibility"` - Forks int `json:"forks"` - OpenIssues int `json:"open_issues"` - Watchers int `json:"watchers"` - DefaultBranch string `json:"default_branch"` - TempCloneToken interface{} `json:"temp_clone_token"` - Organization struct { - Login string `json:"login"` - ID int `json:"id"` - NodeID string `json:"node_id"` - AvatarURL string `json:"avatar_url"` - GravatarID string `json:"gravatar_id"` - URL string `json:"url"` - HTMLURL string `json:"html_url"` - FollowersURL string `json:"followers_url"` - FollowingURL string `json:"following_url"` - GistsURL string `json:"gists_url"` - StarredURL string `json:"starred_url"` - SubscriptionsURL string `json:"subscriptions_url"` - OrganizationsURL string `json:"organizations_url"` - ReposURL string `json:"repos_url"` - EventsURL string `json:"events_url"` - ReceivedEventsURL string `json:"received_events_url"` - Type string `json:"type"` - SiteAdmin bool `json:"site_admin"` - } `json:"organization"` - NetworkCount int `json:"network_count"` - SubscribersCount int `json:"subscribers_count"` -} diff --git a/github/updater.go b/github/updater.go index 32e74a8..a6aee0a 100644 --- a/github/updater.go +++ b/github/updater.go @@ -10,7 +10,7 @@ import ( ) type ghUpdateData struct { - ModID string `mapstructure:"mod-id"` // The slug of the repo but named modId for consistency reasons + Slug string `mapstructure:"slug"` InstalledVersion string `mapstructure:"version"` Branch string `mapstructure:"branch"` } @@ -40,7 +40,7 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack data := rawData.(ghUpdateData) - newVersion, err := getLatestVersion(data.ModID, pack, data.Branch) + newVersion, err := getLatestVersion(data.Slug, data.Branch) if err != nil { results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest version: %v", err)} continue @@ -61,7 +61,7 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack results[i] = core.UpdateCheck{ UpdateAvailable: true, UpdateString: mod.FileName + " -> " + newFilename, - CachedState: cachedStateStore{data.ModID, newVersion}, + CachedState: cachedStateStore{data.Slug, newVersion}, } } @@ -91,7 +91,7 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { HashFormat: "sha1", Hash: hash, } - mod.Update["github"]["version"] = version.ID + mod.Update["github"]["version"] = version.TagName } return nil From 6116393310a95560166d856f5791e99f33b2cf7b Mon Sep 17 00:00:00 2001 From: unilock Date: Tue, 30 May 2023 21:01:29 -0400 Subject: [PATCH 08/22] github: fixes and improvements - "file" -> "asset" - "version" -> "tag" or "release" (where appropriate) - fix updater.go for upstream changes - make printed log messages more similar to those of other modules - move http request function(s) to separate file "request.go" - remove the concept of a "Mod"; we're using "Repo"s (GitHub repositories) instead - remove unnecessary fields in structs - use sha256 instead of sha1 for asset checksums Signed-off-by: unilock --- github/github.go | 139 +++++++++++----------------------------------- github/install.go | 83 ++++++++++++--------------- github/request.go | 33 +++++++++++ github/updater.go | 26 ++++----- 4 files changed, 112 insertions(+), 169 deletions(-) create mode 100644 github/request.go diff --git a/github/github.go b/github/github.go index 33881c0..e67c175 100644 --- a/github/github.go +++ b/github/github.go @@ -4,9 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" - "time" "github.com/mitchellh/mapstructure" "github.com/packwiz/packwiz/cmd" @@ -27,15 +26,15 @@ func init() { func fetchRepo(slug string) (Repo, error) { var repo Repo - res, err := http.Get(githubApiUrl + "repos/" + slug) + res, err := http.Get(githubApiUrl + "repos/" + slug) if err != nil { return repo, err } defer res.Body.Close() - repoBody, err := ioutil.ReadAll(res.Body) + repoBody, err := io.ReadAll(res.Body) if err != nil { return repo, err } @@ -44,49 +43,24 @@ func fetchRepo(slug string) (Repo, error) { if err != nil { return repo, err } + + if repo.FullName == "" { + return repo, errors.New("invalid json while fetching mod: " + slug) + } + return repo, nil } -func fetchMod(slug string) (Mod, error) { - - var mod Mod - - repo, err := fetchRepo(slug) - - if err != nil { - return mod, err - } - - release, err := getLatestVersion(slug, "") - - if err != nil { - return mod, err - } - - mod = Mod{ - ID: repo.Name, - Slug: slug, - Team: repo.Owner.Login, - Title: repo.Name, - Description: repo.Description, - Published: repo.CreatedAt, - Updated: release.CreatedAt, - License: repo.License, - ClientSide: "unknown", - ServerSide: "unknown", - Categories: repo.Topics, - } - if mod.ID == "" { - return mod, errors.New("invalid json whilst fetching mod: " + slug) - } - - return mod, nil - +type Repo struct { + ID int `json:"id"` + NodeID string `json:"node_id"` // TODO: use this with GH API, instead of name (to acct. for repo renames?) + store in mod.pw.toml + Name string `json:"name"` // "hello_world" + FullName string `json:"full_name"` // "owner/hello_world" } -type ModReleases struct { +type Release struct { URL string `json:"url"` - NodeID string `json:"node_id"` + NodeID string `json:"node_id"` // TODO: probably also use this with GH API TagName string `json:"tag_name"` TargetCommitish string `json:"target_commitish"` // The branch of the release Name string `json:"name"` @@ -95,30 +69,9 @@ type ModReleases struct { } type Asset struct { - URL string `json:"url"` - Name string `json:"name"` - UpdatedAt time.Time `json:"updated_at"` - BrowserDownloadURL string `json:"browser_download_url"` -} - -type Repo struct { - ID int `json:"id"` - Name string `json:"name"` - FullName string `json:"full_name"` - Owner struct { - Login string `json:"login"` - } `json:"owner"` - Description string `json:"description"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - License struct { - Key string `json:"key"` - Name string `json:"name"` - SpdxID string `json:"spdx_id"` - URL string `json:"url"` - NodeID string `json:"node_id"` - } `json:"license"` - Topics []string `json:"topics"` + URL string `json:"url"` + BrowserDownloadURL string `json:"browser_download_url"` + Name string `json:"name"` } func (u ghUpdateData) ToMap() (map[string]interface{}, error) { @@ -127,58 +80,28 @@ func (u ghUpdateData) ToMap() (map[string]interface{}, error) { return newMap, err } -type License struct { - Id string `json:"id"` //The license id of a mod, retrieved from the licenses get route - Name string `json:"name"` //The long for name of a license - Url string `json:"url"` //The URL to this license -} - -type Mod struct { - ID string `json:"id"` //The ID of the mod, encoded as a base62 string - Slug string `json:"slug"` //The slug of a mod, used for vanity URLs - Team string `json:"team"` //The id of the team that has ownership of this mod - Title string `json:"title"` //The title or name of the mod - Description string `json:"description"` //A short description of the mod - // Body string `json:"body"` //A long form description of the mod. - // BodyUrl string `json:"body_url"` //DEPRECATED The link to the long description of the mod (Optional) - Published string `json:"published"` //The date at which the mod was first published - Updated string `json:"updated"` //The date at which the mod was updated - License struct { - Key string `json:"key"` - Name string `json:"name"` - SpdxID string `json:"spdx_id"` - URL string `json:"url"` - NodeID string `json:"node_id"` - } `json:"license"` - ClientSide string `json:"client_side"` //The support range for the client mod - required, optional, unsupported, or unknown - ServerSide string `json:"server_side"` //The support range for the server mod - required, optional, unsupported, or unknown - // Downloads int `json:"downloads"` //The total number of downloads the mod has - Categories []string `json:"categories"` //A list of the categories that the mod is in - Versions []string `json:"versions"` //A list of ids for versions of the mod - IconUrl string `json:"icon_url"` //The URL of the icon of the mod (Optional) - IssuesUrl string `json:"issues_url"` //An optional link to where to submit bugs or issues with the mod (Optional) - SourceUrl string `json:"source_url"` //An optional link to the source code for the mod (Optional) - WikiUrl string `json:"wiki_url"` //An optional link to the mod's wiki page or other relevant information (Optional) - DiscordUrl string `json:"discord_url"` //An optional link to the mod's discord (Optional) -} - -func (u Asset) getSha1() (string, error) { +func (u Asset) getSha256() (string, error) { // TODO potentionally cache downloads to speed things up and avoid getting ratelimited by github! - mainHasher, err := core.GetHashImpl("sha1") + mainHasher, err := core.GetHashImpl("sha256") + if err != nil { + return "", err + } + resp, err := http.Get(u.BrowserDownloadURL) if err != nil { return "", err } - if resp.StatusCode == 404 { - return "", fmt.Errorf("Asset not found") - } - if resp.StatusCode != 200 { - return "", fmt.Errorf("Invalid response code: %d", resp.StatusCode) + return "", fmt.Errorf("invalid response status: %v", resp.StatusCode) } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + mainHasher.Write(body) hash := mainHasher.Sum(nil) diff --git a/github/install.go b/github/install.go index 7765557..0326465 100644 --- a/github/install.go +++ b/github/install.go @@ -4,8 +4,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" + "io" "os" "path/filepath" "regexp" @@ -16,48 +15,46 @@ import ( "github.com/spf13/viper" ) -var GithubRegex = regexp.MustCompile("https?://(?:www\\.)?github\\.com/([^/]+/[^/]+)") +var GithubRegex = regexp.MustCompile(`^https?://(?:www\.)?github\.com/([^/]+/[^/]+)`) // installCmd represents the install command var installCmd = &cobra.Command{ - Use: "install [mod]", - Short: "Install mods from github releases", - Aliases: []string{"add", "get"}, + Use: "add [URL]", + Short: "Add a project from a GitHub repository URL", + Aliases: []string{"install", "get"}, Args: cobra.ArbitraryArgs, Run: func(cmd *cobra.Command, args []string) { pack, err := core.LoadPack() - if err != nil { fmt.Println(err) os.Exit(1) } if len(args) == 0 || len(args[0]) == 0 { - fmt.Println("You must specify a mod.") + fmt.Println("You must specify a GitHub repository URL.") os.Exit(1) } - //Try interpreting the arg as a modId or slug. - //Modrinth transparently handles slugs/mod ids in their api; we don't have to detect which one it is. + // Try interpreting the argument as a slug, or GitHub repository URL. var slug string - //Try to see if it's a site, if extract the id/slug from the url. - //Otherwise, interpret the arg as a id/slug straight up + // Check if the argument is a valid GitHub repository URL; if so, extract the slug from the URL. + // Otherwise, interpret the argument as a slug directly. matches := GithubRegex.FindStringSubmatch(args[0]) - if matches != nil && len(matches) == 2 { + if len(matches) == 2 { slug = matches[1] } else { slug = args[0] } - mod, err := fetchMod(slug) + repo, err := fetchRepo(slug) if err != nil { fmt.Println("Failed to get the mod ", err) os.Exit(1) } - installMod(mod, pack) + installMod(repo, pack) }, } @@ -67,10 +64,8 @@ func init() { const githubApiUrl = "https://api.github.com/" -func installMod(mod Mod, pack core.Pack) error { - fmt.Printf("Found repo %s: '%s'.\n", mod.Slug, mod.Description) - - latestVersion, err := getLatestVersion(mod.Slug, "") +func installMod(repo Repo, pack core.Pack) error { + latestVersion, err := getLatestVersion(repo.FullName, "") if err != nil { return fmt.Errorf("failed to get latest version: %v", err) } @@ -78,28 +73,20 @@ func installMod(mod Mod, pack core.Pack) error { return errors.New("mod is not available for this Minecraft version (use the acceptable-game-versions option to accept more) or mod loader") } - return installVersion(mod, latestVersion, pack) + return installVersion(repo, latestVersion, pack) } -func getLatestVersion(slug string, branch string) (ModReleases, error) { - var modReleases []ModReleases - var release ModReleases +func getLatestVersion(slug string, branch string) (Release, error) { + var modReleases []Release + var release Release - resp, err := http.Get(githubApiUrl + "repos/" + slug + "/releases") + resp, err := ghDefaultClient.makeGet(slug) if err != nil { return release, err } - if resp.StatusCode == 404 { - return release, fmt.Errorf("mod not found (for URL %v)", githubApiUrl+"repos/"+slug+"/releases") - } - - if resp.StatusCode != 200 { - return release, fmt.Errorf("invalid response status %v for URL %v", resp.Status, githubApiUrl+"repos/"+slug+"/releases") - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return release, err } @@ -116,23 +103,23 @@ func getLatestVersion(slug string, branch string) (ModReleases, error) { return modReleases[0], nil } -func installVersion(mod Mod, version ModReleases, pack core.Pack) error { - var files = version.Assets +func installVersion(repo Repo, release Release, pack core.Pack) error { + var files = release.Assets if len(files) == 0 { - return errors.New("version doesn't have any files attached") + return errors.New("release doesn't have any files attached") } // TODO: add some way to allow users to pick which file to install? var file = files[0] - for _, v := range version.Assets { + for _, v := range release.Assets { if strings.HasSuffix(v.Name, ".jar") { file = v } } //Install the file - fmt.Printf("Installing %s from version %s\n", file.URL, version.Name) + fmt.Printf("Installing %s from release %s\n", file.Name, release.TagName) index, err := pack.LoadIndex() if err != nil { return err @@ -141,26 +128,26 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { updateMap := make(map[string]map[string]interface{}) updateMap["github"], err = ghUpdateData{ - Slug: mod.Slug, - InstalledVersion: version.TagName, - Branch: version.TargetCommitish, + Slug: repo.FullName, + Tag: release.TagName, + Branch: release.TargetCommitish, }.ToMap() if err != nil { return err } - hash, error := file.getSha1() - if error != nil || hash == "" { - return errors.New("file doesn't have a hash") + hash, err := file.getSha256() + if err != nil { + return err } modMeta := core.Mod{ - Name: mod.Title, + Name: repo.Name, FileName: file.Name, - Side: "unknown", + Side: core.UniversalSide, Download: core.ModDownload{ URL: file.BrowserDownloadURL, - HashFormat: "sha1", + HashFormat: "sha256", Hash: hash, }, Update: updateMap, @@ -170,7 +157,7 @@ func installVersion(mod Mod, version ModReleases, pack core.Pack) error { if folder == "" { folder = "mods" } - path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, mod.Title+core.MetaExtension)) + path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, repo.Name+core.MetaExtension)) // If the file already exists, this will overwrite it!!! // TODO: Should this be improved? diff --git a/github/request.go b/github/request.go new file mode 100644 index 0000000..79dd599 --- /dev/null +++ b/github/request.go @@ -0,0 +1,33 @@ +package github + +import ( + "fmt" + "net/http" +) + +const ghApiServer = "api.github.com" + +type ghApiClient struct { + httpClient *http.Client +} + +var ghDefaultClient = ghApiClient{&http.Client{}} + +func (c *ghApiClient) makeGet(slug string) (*http.Response, error) { + req, err := http.NewRequest("GET", "https://" + ghApiServer + "/repos/" + slug + "/releases", nil) + if err != nil { + return nil, err + } + + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("invalid response status: %v", resp.Status) + } + return resp, nil +} diff --git a/github/updater.go b/github/updater.go index a6aee0a..65f99d7 100644 --- a/github/updater.go +++ b/github/updater.go @@ -10,9 +10,9 @@ import ( ) type ghUpdateData struct { - Slug string `mapstructure:"slug"` - InstalledVersion string `mapstructure:"version"` - Branch string `mapstructure:"branch"` + Slug string `mapstructure:"slug"` + Tag string `mapstructure:"tag"` + Branch string `mapstructure:"branch"` } type ghUpdater struct{} @@ -25,16 +25,16 @@ func (u ghUpdater) ParseUpdate(updateUnparsed map[string]interface{}) (interface type cachedStateStore struct { ModID string - Version ModReleases + Version Release } -func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack) ([]core.UpdateCheck, error) { +func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateCheck, error) { results := make([]core.UpdateCheck, len(mods)) for i, mod := range mods { rawData, ok := mod.GetParsedUpdateData("github") if !ok { - results[i] = core.UpdateCheck{Error: errors.New("couldn't parse mod data")} + results[i] = core.UpdateCheck{Error: errors.New("failed to parse update metadata")} continue } @@ -46,13 +46,13 @@ func (u ghUpdater) CheckUpdate(mods []core.Mod, mcVersion string, pack core.Pack continue } - if newVersion.TagName == data.InstalledVersion { //The latest version from the site is the same as the installed one + if newVersion.TagName == data.Tag { // The latest version from the site is the same as the installed one results[i] = core.UpdateCheck{UpdateAvailable: false} continue } if len(newVersion.Assets) == 0 { - results[i] = core.UpdateCheck{Error: errors.New("new version doesn't have any files")} + results[i] = core.UpdateCheck{Error: errors.New("new version doesn't have any assets")} continue } @@ -80,18 +80,18 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { } } - hash, error := file.getSha1() - if error != nil || hash == "" { - return errors.New("file doesn't have a hash") + hash, err := file.getSha256() + if err != nil { + return err } mod.FileName = file.Name mod.Download = core.ModDownload{ URL: file.BrowserDownloadURL, - HashFormat: "sha1", + HashFormat: "sha256", Hash: hash, } - mod.Update["github"]["version"] = version.TagName + mod.Update["github"]["tag"] = version.TagName } return nil From 01945213d774ebc666378641699843a1d3c06ca1 Mon Sep 17 00:00:00 2001 From: unilock Date: Wed, 31 May 2023 20:19:56 -0400 Subject: [PATCH 09/22] github: standardize language + fix api requests language: - funcs, vars, log messages, etc. api requests: - make all requests use the makeGet() function - include packwiz user agent Signed-off-by: unilock --- github/github.go | 15 ++++---------- github/install.go | 51 ++++++++++++++--------------------------------- github/request.go | 27 ++++++++++++++++++++++--- github/updater.go | 34 ++++++++++++++++++------------- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/github/github.go b/github/github.go index e67c175..60bbb9c 100644 --- a/github/github.go +++ b/github/github.go @@ -3,9 +3,7 @@ package github import ( "encoding/json" "errors" - "fmt" "io" - "net/http" "github.com/mitchellh/mapstructure" "github.com/packwiz/packwiz/cmd" @@ -16,7 +14,7 @@ import ( var githubCmd = &cobra.Command{ Use: "github", Aliases: []string{"gh"}, - Short: "Manage github-based mods", + Short: "Manage projects released on GitHub", } func init() { @@ -27,7 +25,7 @@ func init() { func fetchRepo(slug string) (Repo, error) { var repo Repo - res, err := http.Get(githubApiUrl + "repos/" + slug) + res, err := ghDefaultClient.getRepo(slug) if err != nil { return repo, err } @@ -45,7 +43,7 @@ func fetchRepo(slug string) (Repo, error) { } if repo.FullName == "" { - return repo, errors.New("invalid json while fetching mod: " + slug) + return repo, errors.New("invalid json while fetching project: " + slug) } return repo, nil @@ -53,14 +51,12 @@ func fetchRepo(slug string) (Repo, error) { type Repo struct { ID int `json:"id"` - NodeID string `json:"node_id"` // TODO: use this with GH API, instead of name (to acct. for repo renames?) + store in mod.pw.toml Name string `json:"name"` // "hello_world" FullName string `json:"full_name"` // "owner/hello_world" } type Release struct { URL string `json:"url"` - NodeID string `json:"node_id"` // TODO: probably also use this with GH API TagName string `json:"tag_name"` TargetCommitish string `json:"target_commitish"` // The branch of the release Name string `json:"name"` @@ -87,13 +83,10 @@ func (u Asset) getSha256() (string, error) { return "", err } - resp, err := http.Get(u.BrowserDownloadURL) + resp, err := ghDefaultClient.makeGet(u.BrowserDownloadURL) if err != nil { return "", err } - if resp.StatusCode != 200 { - return "", fmt.Errorf("invalid response status: %v", resp.StatusCode) - } defer resp.Body.Close() diff --git a/github/install.go b/github/install.go index 0326465..89f38dc 100644 --- a/github/install.go +++ b/github/install.go @@ -19,8 +19,8 @@ var GithubRegex = regexp.MustCompile(`^https?://(?:www\.)?github\.com/([^/]+/[^/ // installCmd represents the install command var installCmd = &cobra.Command{ - Use: "add [URL]", - Short: "Add a project from a GitHub repository URL", + Use: "add [URL|slug]", + Short: "Add a project from a GitHub repository URL or slug", Aliases: []string{"install", "get"}, Args: cobra.ArbitraryArgs, Run: func(cmd *cobra.Command, args []string) { @@ -50,7 +50,7 @@ var installCmd = &cobra.Command{ repo, err := fetchRepo(slug) if err != nil { - fmt.Println("Failed to get the mod ", err) + fmt.Printf("Failed to add project: %s\n", err) os.Exit(1) } @@ -62,25 +62,20 @@ func init() { githubCmd.AddCommand(installCmd) } -const githubApiUrl = "https://api.github.com/" - func installMod(repo Repo, pack core.Pack) error { - latestVersion, err := getLatestVersion(repo.FullName, "") + latestRelease, err := getLatestRelease(repo.FullName, "") if err != nil { - return fmt.Errorf("failed to get latest version: %v", err) - } - if latestVersion.URL == "" { - return errors.New("mod is not available for this Minecraft version (use the acceptable-game-versions option to accept more) or mod loader") + return fmt.Errorf("failed to get latest release: %v", err) } - return installVersion(repo, latestVersion, pack) + return installRelease(repo, latestRelease, pack) } -func getLatestVersion(slug string, branch string) (Release, error) { - var modReleases []Release +func getLatestRelease(slug string, branch string) (Release, error) { + var releases []Release var release Release - resp, err := ghDefaultClient.makeGet(slug) + resp, err := ghDefaultClient.getReleases(slug) if err != nil { return release, err } @@ -90,20 +85,20 @@ func getLatestVersion(slug string, branch string) (Release, error) { if err != nil { return release, err } - err = json.Unmarshal(body, &modReleases) + err = json.Unmarshal(body, &releases) if err != nil { return release, err } - for _, r := range modReleases { + for _, r := range releases { if r.TargetCommitish == branch { return r, nil } } - return modReleases[0], nil + return releases[0], nil } -func installVersion(repo Repo, release Release, pack core.Pack) error { +func installRelease(repo Repo, release Release, pack core.Pack) error { var files = release.Assets if len(files) == 0 { @@ -157,7 +152,7 @@ func installVersion(repo Repo, release Release, pack core.Pack) error { if folder == "" { folder = "mods" } - path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, repo.Name+core.MetaExtension)) + path = modMeta.SetMetaPath(filepath.Join(viper.GetString("meta-folder-base"), folder, core.SlugifyName(repo.Name)+core.MetaExtension)) // If the file already exists, this will overwrite it!!! // TODO: Should this be improved? @@ -168,21 +163,5 @@ func installVersion(repo Repo, release Release, pack core.Pack) error { if err != nil { return err } - err = index.RefreshFileWithHash(path, format, hash, true) - if err != nil { - return err - } - err = index.Write() - if err != nil { - return err - } - err = pack.UpdateIndexHash() - if err != nil { - return err - } - err = pack.Write() - if err != nil { - return err - } - return nil + return index.RefreshFileWithHash(path, format, hash, true) } diff --git a/github/request.go b/github/request.go index 79dd599..6c33887 100644 --- a/github/request.go +++ b/github/request.go @@ -3,6 +3,8 @@ package github import ( "fmt" "net/http" + + "github.com/packwiz/packwiz/core" ) const ghApiServer = "api.github.com" @@ -13,21 +15,40 @@ type ghApiClient struct { var ghDefaultClient = ghApiClient{&http.Client{}} -func (c *ghApiClient) makeGet(slug string) (*http.Response, error) { - req, err := http.NewRequest("GET", "https://" + ghApiServer + "/repos/" + slug + "/releases", nil) +func (c *ghApiClient) makeGet(url string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } + req.Header.Set("User-Agent", core.UserAgent) req.Header.Set("Accept", "application/vnd.github+json") resp, err := c.httpClient.Do(req) if err != nil { return nil, err } - if resp.StatusCode != 200 { return nil, fmt.Errorf("invalid response status: %v", resp.Status) } + + return resp, nil +} + +func (c *ghApiClient) getRepo(slug string) (*http.Response, error) { + resp, err := c.makeGet("https://" + ghApiServer + "/repos/" + slug) + if err != nil { + return resp, err + } + + return resp, nil +} + +func (c *ghApiClient) getReleases(slug string) (*http.Response, error) { + resp, err := c.getRepo(slug + "/releases") + if err != nil { + return resp, err + } + return resp, nil } diff --git a/github/updater.go b/github/updater.go index 65f99d7..157d067 100644 --- a/github/updater.go +++ b/github/updater.go @@ -24,8 +24,8 @@ func (u ghUpdater) ParseUpdate(updateUnparsed map[string]interface{}) (interface } type cachedStateStore struct { - ModID string - Version Release + Slug string + Release Release } func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateCheck, error) { @@ -40,28 +40,33 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC data := rawData.(ghUpdateData) - newVersion, err := getLatestVersion(data.Slug, data.Branch) + newRelease, err := getLatestRelease(data.Slug, data.Branch) if err != nil { - results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest version: %v", err)} + results[i] = core.UpdateCheck{Error: fmt.Errorf("failed to get latest release: %v", err)} continue } - if newVersion.TagName == data.Tag { // The latest version from the site is the same as the installed one + if newRelease.TagName == data.Tag { // The latest release is the same as the installed one results[i] = core.UpdateCheck{UpdateAvailable: false} continue } - if len(newVersion.Assets) == 0 { - results[i] = core.UpdateCheck{Error: errors.New("new version doesn't have any assets")} + if len(newRelease.Assets) == 0 { + results[i] = core.UpdateCheck{Error: errors.New("new release doesn't have any assets")} continue } - newFilename := newVersion.Assets[0].Name + newFile := newRelease.Assets[0] + for _, v := range newRelease.Assets { + if strings.HasSuffix(v.Name, ".jar") { + newFile = v + } + } results[i] = core.UpdateCheck{ UpdateAvailable: true, - UpdateString: mod.FileName + " -> " + newFilename, - CachedState: cachedStateStore{data.Slug, newVersion}, + UpdateString: mod.FileName + " -> " + newFile.Name, + CachedState: cachedStateStore{data.Slug, newRelease}, } } @@ -71,10 +76,11 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { for i, mod := range mods { modState := cachedState[i].(cachedStateStore) - var version = modState.Version + var release = modState.Release - var file = version.Assets[0] - for _, v := range version.Assets { + // yes, this is duplicated - i guess we should just cache asset + tag instead of entire release...? + var file = release.Assets[0] + for _, v := range release.Assets { if strings.HasSuffix(v.Name, ".jar") { file = v } @@ -91,7 +97,7 @@ func (u ghUpdater) DoUpdate(mods []*core.Mod, cachedState []interface{}) error { HashFormat: "sha256", Hash: hash, } - mod.Update["github"]["tag"] = version.TagName + mod.Update["github"]["tag"] = release.TagName } return nil From c96a6a30add211af3617a76d3b50f55f7f40a2ac Mon Sep 17 00:00:00 2001 From: unilock Date: Wed, 31 May 2023 20:54:35 -0400 Subject: [PATCH 10/22] github: don't return releases on wrong branch if a branch is specified, but a release cannot be found on that branch, return an error instead of simply the latest release Signed-off-by: unilock --- github/install.go | 15 ++++++++++----- github/request.go | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/github/install.go b/github/install.go index 89f38dc..55a4127 100644 --- a/github/install.go +++ b/github/install.go @@ -85,14 +85,19 @@ func getLatestRelease(slug string, branch string) (Release, error) { if err != nil { return release, err } + err = json.Unmarshal(body, &releases) if err != nil { return release, err } - for _, r := range releases { - if r.TargetCommitish == branch { - return r, nil + + if branch != "" { + for _, r := range releases { + if r.TargetCommitish == branch { + return r, nil + } } + return release, fmt.Errorf("failed to find release for branch %v", branch) } return releases[0], nil @@ -113,7 +118,7 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { } } - //Install the file + // Install the file fmt.Printf("Installing %s from release %s\n", file.Name, release.TagName) index, err := pack.LoadIndex() if err != nil { @@ -125,7 +130,7 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { updateMap["github"], err = ghUpdateData{ Slug: repo.FullName, Tag: release.TagName, - Branch: release.TargetCommitish, + Branch: release.TargetCommitish, // TODO: if no branch is specified by the user, we shouldn't record it - in order to remain branch-agnostic in getLatestRelease() }.ToMap() if err != nil { return err diff --git a/github/request.go b/github/request.go index 6c33887..f3fb5f6 100644 --- a/github/request.go +++ b/github/request.go @@ -7,6 +7,7 @@ import ( "github.com/packwiz/packwiz/core" ) +// TODO: allow setting github api key via env variable const ghApiServer = "api.github.com" type ghApiClient struct { From 8d4f8d9a8600eb62891767293580b2b4f5275013 Mon Sep 17 00:00:00 2001 From: unilock Date: Thu, 1 Jun 2023 12:54:32 -0400 Subject: [PATCH 11/22] github: Accept GitHub API token via OS environment variable "GH_API_TOKEN" Signed-off-by: unilock --- github/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/github/request.go b/github/request.go index f3fb5f6..28b2119 100644 --- a/github/request.go +++ b/github/request.go @@ -3,6 +3,7 @@ package github import ( "fmt" "net/http" + "os" "github.com/packwiz/packwiz/core" ) @@ -17,6 +18,8 @@ type ghApiClient struct { var ghDefaultClient = ghApiClient{&http.Client{}} func (c *ghApiClient) makeGet(url string) (*http.Response, error) { + ghApiToken := os.Getenv("GH_API_TOKEN") + req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err @@ -24,6 +27,9 @@ func (c *ghApiClient) makeGet(url string) (*http.Response, error) { req.Header.Set("User-Agent", core.UserAgent) req.Header.Set("Accept", "application/vnd.github+json") + if ghApiToken != "" { + req.Header.Set("Authorization", "Bearer " + ghApiToken) + } resp, err := c.httpClient.Do(req) if err != nil { From d7fd884f48ed559226d5f3dbdf6f8a8056e28fd1 Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 11 Sep 2023 09:11:14 -0400 Subject: [PATCH 12/22] github: Use Viper for GitHub API token Signed-off-by: unilock --- github/request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github/request.go b/github/request.go index 28b2119..5b1a429 100644 --- a/github/request.go +++ b/github/request.go @@ -3,9 +3,9 @@ package github import ( "fmt" "net/http" - "os" "github.com/packwiz/packwiz/core" + "github.com/spf13/viper" ) // TODO: allow setting github api key via env variable @@ -18,7 +18,7 @@ type ghApiClient struct { var ghDefaultClient = ghApiClient{&http.Client{}} func (c *ghApiClient) makeGet(url string) (*http.Response, error) { - ghApiToken := os.Getenv("GH_API_TOKEN") + ghApiToken := viper.GetString("github.token") req, err := http.NewRequest("GET", url, nil) if err != nil { From e5d1d9d383bf39edf2423cdbb88b2fee620346dc Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 11 Sep 2023 09:59:19 -0400 Subject: [PATCH 13/22] github: Properly handle writing to pack index Signed-off-by: unilock --- github/install.go | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/github/install.go b/github/install.go index 55a4127..76cd8d4 100644 --- a/github/install.go +++ b/github/install.go @@ -54,7 +54,11 @@ var installCmd = &cobra.Command{ os.Exit(1) } - installMod(repo, pack) + err = installMod(repo, pack) + if err != nil { + fmt.Printf("Failed to add project: %s\n", err) + os.Exit(1) + } }, } @@ -168,5 +172,25 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { if err != nil { return err } - return index.RefreshFileWithHash(path, format, hash, true) + + err = index.RefreshFileWithHash(path, format, hash, true) + if err != nil { + return err + } + err = index.Write() + if err != nil { + return err + } + err = pack.UpdateIndexHash() + if err != nil { + return err + } + err = pack.Write() + if err != nil { + return err + } + + fmt.Printf("Project \"%s\" successfully added! (%s)\n", repo.Name, file.Name) + return nil +} } From 3f5b953d00f2363dfed7a8ee5ae3339f7b299b74 Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 11 Sep 2023 10:00:04 -0400 Subject: [PATCH 14/22] github: Allow specifying branch to retrieve releases for via CLI Signed-off-by: unilock --- github/install.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/github/install.go b/github/install.go index 76cd8d4..add69e9 100644 --- a/github/install.go +++ b/github/install.go @@ -37,6 +37,7 @@ var installCmd = &cobra.Command{ // Try interpreting the argument as a slug, or GitHub repository URL. var slug string + var branch string // Check if the argument is a valid GitHub repository URL; if so, extract the slug from the URL. // Otherwise, interpret the argument as a slug directly. @@ -54,7 +55,11 @@ var installCmd = &cobra.Command{ os.Exit(1) } - err = installMod(repo, pack) + if branchFlag != "" { + branch = branchFlag + } + + err = installMod(repo, branch, pack) if err != nil { fmt.Printf("Failed to add project: %s\n", err) os.Exit(1) @@ -62,12 +67,8 @@ var installCmd = &cobra.Command{ }, } -func init() { - githubCmd.AddCommand(installCmd) -} - -func installMod(repo Repo, pack core.Pack) error { - latestRelease, err := getLatestRelease(repo.FullName, "") +func installMod(repo Repo, branch string, pack core.Pack) error { + latestRelease, err := getLatestRelease(repo.FullName, branch) if err != nil { return fmt.Errorf("failed to get latest release: %v", err) } @@ -193,4 +194,11 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { fmt.Printf("Project \"%s\" successfully added! (%s)\n", repo.Name, file.Name) return nil } + +var branchFlag string + +func init() { + githubCmd.AddCommand(installCmd) + + installCmd.Flags().StringVar(&branchFlag, "branch", "", "The GitHub repository branch to retrieve releases for") } From 3859b37267ed1d66fccb7591104a1023556a8917 Mon Sep 17 00:00:00 2001 From: unilock Date: Thu, 14 Sep 2023 19:54:37 -0400 Subject: [PATCH 15/22] github: Notify user when encountering GitHub API ratelimit Signed-off-by: unilock --- github/request.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/github/request.go b/github/request.go index 5b1a429..6b6b10f 100644 --- a/github/request.go +++ b/github/request.go @@ -3,6 +3,7 @@ package github import ( "fmt" "net/http" + "strconv" "github.com/packwiz/packwiz/core" "github.com/spf13/viper" @@ -35,10 +36,24 @@ func (c *ghApiClient) makeGet(url string) (*http.Response, error) { if err != nil { return nil, err } + + ratelimit, err := strconv.Atoi(resp.Header.Get("x-ratelimit-remaining")) + if err != nil { + return nil, err + } + + if resp.StatusCode == 403 && ratelimit == 0 { + return nil, fmt.Errorf("GitHub API ratelimit exceeded; time of reset: %v", resp.Header.Get("x-ratelimit-reset")) + } if resp.StatusCode != 200 { return nil, fmt.Errorf("invalid response status: %v", resp.Status) } + if ratelimit < 10 { + fmt.Printf("Warning: GitHub API allows %v more requests before ratelimiting\n", ratelimit) + fmt.Println("Specifying a token is recommended; see documentation") + } + return resp, nil } From 72afdee4d878dadfbe098673b1eb4a1a73c81f69 Mon Sep 17 00:00:00 2001 From: unilock Date: Thu, 26 Oct 2023 10:56:08 -0400 Subject: [PATCH 16/22] github: Handle empty x-ratelimit-remaining header Signed-off-by: unilock --- github/request.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/github/request.go b/github/request.go index 6b6b10f..93f8cfa 100644 --- a/github/request.go +++ b/github/request.go @@ -37,9 +37,15 @@ func (c *ghApiClient) makeGet(url string) (*http.Response, error) { return nil, err } - ratelimit, err := strconv.Atoi(resp.Header.Get("x-ratelimit-remaining")) - if err != nil { - return nil, err + // TODO: there is likely a better way to do this + ratelimit := 999 + + ratelimit_header := resp.Header.Get("x-ratelimit-remaining") + if ratelimit_header != "" { + ratelimit, err = strconv.Atoi(ratelimit_header) + if err != nil { + return nil, err + } } if resp.StatusCode == 403 && ratelimit == 0 { From bae4a6be644dc40990784f45add24fda679280a8 Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 15 Apr 2024 15:29:06 -0400 Subject: [PATCH 17/22] github: remove outdated comment Signed-off-by: unilock --- github/request.go | 1 - 1 file changed, 1 deletion(-) diff --git a/github/request.go b/github/request.go index 93f8cfa..7604ffe 100644 --- a/github/request.go +++ b/github/request.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/viper" ) -// TODO: allow setting github api key via env variable const ghApiServer = "api.github.com" type ghApiClient struct { From d54da349d5542a1c5a730fb10a992249590d7116 Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 15 Apr 2024 15:30:22 -0400 Subject: [PATCH 18/22] github: allow using a regular expression to match assets safeguard against "undefined behavior" when a release has more than one asset Signed-off-by: unilock --- github/install.go | 45 +++++++++++++++++++++++++++++++++------------ github/updater.go | 19 ++++++++++++++++--- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/github/install.go b/github/install.go index add69e9..ea862f4 100644 --- a/github/install.go +++ b/github/install.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "regexp" - "strings" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" @@ -39,6 +38,15 @@ var installCmd = &cobra.Command{ var slug string var branch string + // Regex to match potential release assets against. + // The default will match any asset with a name that does *not* end with: + // - "-api.jar" + // - "-dev.jar" + // - "-sources.jar" + // In most cases, this will only match one asset. + // TODO: Hopefully. + regex := `^.+(? 1 { + // TODO: also print file names + return errors.New("release has more than one asset matching regex") + } + + file := files[0] + // Install the file fmt.Printf("Installing %s from release %s\n", file.Name, release.TagName) index, err := pack.LoadIndex() @@ -136,6 +154,7 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { Slug: repo.FullName, Tag: release.TagName, Branch: release.TargetCommitish, // TODO: if no branch is specified by the user, we shouldn't record it - in order to remain branch-agnostic in getLatestRelease() + Regex: regex, // TODO: ditto! }.ToMap() if err != nil { return err @@ -196,9 +215,11 @@ func installRelease(repo Repo, release Release, pack core.Pack) error { } var branchFlag string +var regexFlag string func init() { githubCmd.AddCommand(installCmd) installCmd.Flags().StringVar(&branchFlag, "branch", "", "The GitHub repository branch to retrieve releases for") + installCmd.Flags().StringVar(®exFlag, "regex", "", "The regular expression to match releases against") } diff --git a/github/updater.go b/github/updater.go index 157d067..9f66cb8 100644 --- a/github/updater.go +++ b/github/updater.go @@ -3,6 +3,7 @@ package github import ( "errors" "fmt" + "regexp" "strings" "github.com/mitchellh/mapstructure" @@ -13,6 +14,7 @@ type ghUpdateData struct { Slug string `mapstructure:"slug"` Tag string `mapstructure:"tag"` Branch string `mapstructure:"branch"` + Regex string `mapstructure:"regex"` } type ghUpdater struct{} @@ -51,18 +53,29 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC continue } + expr := regexp.MustCompile(data.Regex) + if len(newRelease.Assets) == 0 { results[i] = core.UpdateCheck{Error: errors.New("new release doesn't have any assets")} continue } - newFile := newRelease.Assets[0] + var newFiles []Asset + for _, v := range newRelease.Assets { - if strings.HasSuffix(v.Name, ".jar") { - newFile = v + if expr.MatchString(v.Name) { + newFiles = append(newFiles, v) } } + if len(newFiles) > 1 { + // TODO: also print file names + results[i] = core.UpdateCheck{Error: errors.New("release has more than one asset matching regex")} + continue + } + + newFile := newFiles[0] + results[i] = core.UpdateCheck{ UpdateAvailable: true, UpdateString: mod.FileName + " -> " + newFile.Name, From 979b606b1ea08861d3b1aff61b25c49b13f03884 Mon Sep 17 00:00:00 2001 From: unilock Date: Mon, 15 Apr 2024 19:18:05 -0400 Subject: [PATCH 19/22] github: check + fail if there are no assets matching regex Signed-off-by: unilock --- github/install.go | 4 ++++ github/updater.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/github/install.go b/github/install.go index ea862f4..e47ca62 100644 --- a/github/install.go +++ b/github/install.go @@ -134,6 +134,10 @@ func installRelease(repo Repo, release Release, regex string, pack core.Pack) er } } + if len(files) == 0 { + return errors.New("release doesn't have any assets matching regex") + } + if len(files) > 1 { // TODO: also print file names return errors.New("release has more than one asset matching regex") diff --git a/github/updater.go b/github/updater.go index 9f66cb8..c970961 100644 --- a/github/updater.go +++ b/github/updater.go @@ -68,6 +68,11 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC } } + if len(newFiles) == 0 { + results[i] = core.UpdateCheck{Error: errors.New("release doesn't have any assets matching regex")} + continue + } + if len(newFiles) > 1 { // TODO: also print file names results[i] = core.UpdateCheck{Error: errors.New("release has more than one asset matching regex")} From 7207d4c6a4195e21bfcd620d49e4768e9cb278d9 Mon Sep 17 00:00:00 2001 From: unilock Date: Tue, 16 Apr 2024 07:04:13 -0400 Subject: [PATCH 20/22] github: use regexp2 for advanced regex Signed-off-by: unilock --- github/install.go | 6 ++++-- github/updater.go | 7 ++++--- go.mod | 1 + go.sum | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/github/install.go b/github/install.go index e47ca62..30e535c 100644 --- a/github/install.go +++ b/github/install.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" + "github.com/dlclark/regexp2" "github.com/packwiz/packwiz/core" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -120,7 +121,7 @@ func getLatestRelease(slug string, branch string) (Release, error) { } func installRelease(repo Repo, release Release, regex string, pack core.Pack) error { - expr := regexp.MustCompile(regex) + expr := regexp2.MustCompile(regex, 0) if len(release.Assets) == 0 { return errors.New("release doesn't have any assets attached") @@ -129,7 +130,8 @@ func installRelease(repo Repo, release Release, regex string, pack core.Pack) er var files []Asset for _, v := range release.Assets { - if expr.MatchString(v.Name) { + bl, _ := expr.MatchString(v.Name) + if bl { files = append(files, v) } } diff --git a/github/updater.go b/github/updater.go index c970961..039388d 100644 --- a/github/updater.go +++ b/github/updater.go @@ -3,9 +3,9 @@ package github import ( "errors" "fmt" - "regexp" "strings" + "github.com/dlclark/regexp2" "github.com/mitchellh/mapstructure" "github.com/packwiz/packwiz/core" ) @@ -53,7 +53,7 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC continue } - expr := regexp.MustCompile(data.Regex) + expr := regexp2.MustCompile(data.Regex, 0) if len(newRelease.Assets) == 0 { results[i] = core.UpdateCheck{Error: errors.New("new release doesn't have any assets")} @@ -63,7 +63,8 @@ func (u ghUpdater) CheckUpdate(mods []*core.Mod, pack core.Pack) ([]core.UpdateC var newFiles []Asset for _, v := range newRelease.Assets { - if expr.MatchString(v.Name) { + bl, _ := expr.MatchString(v.Name) + if bl { newFiles = append(newFiles, v) } } diff --git a/go.mod b/go.mod index 7692e9f..0a71260 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aviddiviner/go-murmur v0.0.0-20150519214947-b9740d71e571 github.com/daviddengcn/go-colortext v1.0.0 // indirect + github.com/dlclark/regexp2 v1.11.0 github.com/fatih/camelcase v1.0.0 github.com/igorsobreira/titlecase v0.0.0-20140109233139-4156b5b858ac github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index aed0b7c..ada5dd2 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 4853d807e112791a9b4066859a45a55becdd7bff Mon Sep 17 00:00:00 2001 From: unilock Date: Tue, 16 Apr 2024 07:16:41 -0400 Subject: [PATCH 21/22] github: also exclude -dev-preshadow.jar assets by default seems common among GTNewHorizons projects, which is the primary target (for me personally) Signed-off-by: unilock --- github/install.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/github/install.go b/github/install.go index 30e535c..ab6a777 100644 --- a/github/install.go +++ b/github/install.go @@ -43,10 +43,11 @@ var installCmd = &cobra.Command{ // The default will match any asset with a name that does *not* end with: // - "-api.jar" // - "-dev.jar" + // - "-dev-preshadow.jar" // - "-sources.jar" // In most cases, this will only match one asset. // TODO: Hopefully. - regex := `^.+(? Date: Wed, 28 Aug 2024 18:26:29 +0100 Subject: [PATCH 22/22] Run go fmt ./... --- github/install.go | 2 +- github/request.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/github/install.go b/github/install.go index ab6a777..271f302 100644 --- a/github/install.go +++ b/github/install.go @@ -161,7 +161,7 @@ func installRelease(repo Repo, release Release, regex string, pack core.Pack) er Slug: repo.FullName, Tag: release.TagName, Branch: release.TargetCommitish, // TODO: if no branch is specified by the user, we shouldn't record it - in order to remain branch-agnostic in getLatestRelease() - Regex: regex, // TODO: ditto! + Regex: regex, // TODO: ditto! }.ToMap() if err != nil { return err diff --git a/github/request.go b/github/request.go index 7604ffe..ae9d02a 100644 --- a/github/request.go +++ b/github/request.go @@ -28,7 +28,7 @@ func (c *ghApiClient) makeGet(url string) (*http.Response, error) { req.Header.Set("User-Agent", core.UserAgent) req.Header.Set("Accept", "application/vnd.github+json") if ghApiToken != "" { - req.Header.Set("Authorization", "Bearer " + ghApiToken) + req.Header.Set("Authorization", "Bearer "+ghApiToken) } resp, err := c.httpClient.Do(req)