From d7e916e558a49a0b47b74ed86a45a0c51e2c0cb8 Mon Sep 17 00:00:00 2001 From: comp500 Date: Fri, 26 Apr 2019 17:16:29 +0100 Subject: [PATCH] Index handling, hash calculation --- core/index.go | 163 +++++++++++++++++++++++++++++++++++++++++--------- core/pack.go | 56 +++++++++++++++++ main.go | 15 ++++- 3 files changed, 204 insertions(+), 30 deletions(-) diff --git a/core/index.go b/core/index.go index 0c8e9ea..5635e53 100644 --- a/core/index.go +++ b/core/index.go @@ -1,7 +1,11 @@ package core import ( + "crypto/sha256" + "encoding/hex" + "io" "io/ioutil" "os" + "path/filepath" "sort" "github.com/BurntSushi/toml" @@ -9,24 +13,24 @@ import ( // Index is a representation of the index.toml file for referencing all the files in a pack. type Index struct { - HashFormat string `toml:"hash-format"` - Files []struct { - File string `toml:"file"` - Hash string `toml:"hash"` - HashFormat string `toml:"hash-format,omitempty"` - Alias string `toml:"alias,omitempty"` - } `toml:"files"` - flags Flags - indexFile string + HashFormat string `toml:"hash-format"` + Files []IndexFile `toml:"files"` + flags Flags + indexFile string } -// LoadIndex loads the index file -func LoadIndex(flags Flags) (Index, error) { - indexFile, err := ResolveIndex(flags) - if err != nil { - return Index{}, err - } +// IndexFile is a file in the index +type IndexFile struct { + File string `toml:"file"` + Hash string `toml:"hash"` + HashFormat string `toml:"hash-format,omitempty"` + Alias string `toml:"alias,omitempty"` + MetaFile bool `toml:"metafile,omitempty"` // True when it is a .toml metadata file + fileExistsTemp bool +} +// LoadIndex attempts to load the index file from a path +func LoadIndex(indexFile string, flags Flags) (Index, error) { data, err := ioutil.ReadFile(indexFile) if err != nil { return Index{}, err @@ -37,39 +41,140 @@ func LoadIndex(flags Flags) (Index, error) { } index.flags = flags index.indexFile = indexFile + if len(index.HashFormat) == 0 { + index.HashFormat = "sha256" + } return index, nil } // RemoveFile removes a file from the index. -func (in Index) RemoveFile(path string) { +func (in *Index) RemoveFile(path string) error { + relPath, err := filepath.Rel(filepath.Dir(in.indexFile), path) + if err != nil { + return err + } newFiles := in.Files[:0] for _, v := range in.Files { - if v.File != path { + if filepath.Clean(v.File) != relPath { newFiles = append(newFiles, v) } } in.Files = newFiles + return nil } // resortIndex sorts Files by file name -func (in Index) resortIndex() { +func (in *Index) resortIndex() { sort.SliceStable(in.Files, func(i, j int) bool { - // Compare by alias if names are equal? - // Remove duplicated entries? (compound key on file/alias?) + // TODO: Compare by alias if names are equal? + // TODO: Remove duplicated entries? (compound key on file/alias?) return in.Files[i].File < in.Files[j].File }) } +// updateFile calculates the hash for a given path relative to the pack folder, +// and updates it in the index +func (in *Index) updateFile(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + // Hash usage strategy (may change): + // Just use SHA256, overwrite existing hash regardless of what it is + // May update later to continue using the same hash that was already being used + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return err + } + hashString := hex.EncodeToString(h.Sum(nil)) + + // Find in index + found := false + relPath, err := filepath.Rel(filepath.Dir(in.indexFile), path) + if err != nil { + return err + } + for k, v := range in.Files { + if filepath.Clean(v.File) == relPath { + found = true + // Update hash + in.Files[k].Hash = hashString + if in.HashFormat == "sha256" { + in.Files[k].HashFormat = "" + } else { + in.Files[k].HashFormat = "sha256" + } + // Mark this file as found + in.Files[k].fileExistsTemp = true + // Clean up path if it's untidy + in.Files[k].File = relPath + // Don't break out of loop, as there may be aliased versions that + // also need to be updated + } + } + if !found { + newFile := IndexFile{ + File: relPath, + Hash: hashString, + fileExistsTemp: true, + } + // Override hash format for this file, if the whole index isn't sha256 + if in.HashFormat != "sha256" { + newFile.HashFormat = "sha256" + } + in.Files = append(in.Files, newFile) + } + + return nil +} + // Refresh updates the hashes of all the files in the index, and adds new files to the index -func (in Index) Refresh() error { - // TODO: implement - // process: - // enumerate files, exclude index and pack.toml - // hash them - // check if they exist in list - // if exists, modify existing entry(ies) - // if not exists, add new entry - // resort +func (in *Index) Refresh() error { + // TODO: enumerate existing files, check if they exist (remove if they don't) + + // TODO: If needed, multithreaded hashing + // for i := 0; i < runtime.NumCPU(); i++ {} + + // Get fileinfos of pack.toml and index to compare them + pathPF, _ := filepath.Abs(in.flags.PackFile) + pathIndex, _ := filepath.Abs(in.indexFile) + + // TODO: A method of specifying pack root directory? + // TODO: A method of excluding files + packRoot := filepath.Dir(in.flags.PackFile) + err := filepath.Walk(packRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + // TODO: Handle errors on individual files properly + return err + } + // Exit if the files are the same as the pack/index files + absPath, _ := filepath.Abs(path) + if absPath == pathPF || absPath == pathIndex { + return nil + } + // Exit if this is a directory + if info.IsDir() { + return nil + } + + return in.updateFile(path) + }) + if err != nil { + return err + } + + // Check all the files exist, remove them if they don't + i := 0 + for _, file := range in.Files { + if file.fileExistsTemp { + // Keep file if it exists (already checked in updateFile) + in.Files[i] = file + i++ + } + } + in.Files = in.Files[:i] in.resortIndex() return nil } diff --git a/core/pack.go b/core/pack.go index fadd0a4..3be83c8 100644 --- a/core/pack.go +++ b/core/pack.go @@ -1,6 +1,11 @@ package core import ( + "crypto/sha256" + "encoding/hex" + "io" "io/ioutil" + "os" + "path/filepath" "github.com/BurntSushi/toml" ) @@ -16,6 +21,7 @@ type Pack struct { Versions map[string]string `toml:"versions"` Client map[string]toml.Primitive `toml:"client"` Server map[string]toml.Primitive `toml:"server"` + flags Flags } // LoadPack loads the modpack metadata to a Pack struct @@ -32,6 +38,56 @@ func LoadPack(flags Flags) (Pack, error) { if len(modpack.Index.File) == 0 { modpack.Index.File = "index.toml" } + modpack.flags = flags return modpack, nil } +// LoadIndex attempts to load the index file of this modpack +func (pack Pack) LoadIndex() (Index, error) { + if filepath.IsAbs(pack.Index.File) { + return LoadIndex(pack.Index.File, pack.flags) + } + return LoadIndex(filepath.Join(filepath.Dir(pack.flags.PackFile), pack.Index.File), pack.flags) +} + +// UpdateIndexHash recalculates the hash of the index file of this modpack +func (pack *Pack) UpdateIndexHash() error { + indexFile := filepath.Join(filepath.Dir(pack.flags.PackFile), pack.Index.File) + if filepath.IsAbs(pack.Index.File) { + indexFile = pack.Index.File + } + + f, err := os.Open(indexFile) + if err != nil { + return err + } + defer f.Close() + + // Hash usage strategy (may change): + // Just use SHA256, overwrite existing hash regardless of what it is + // May update later to continue using the same hash that was already being used + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return err + } + hashString := hex.EncodeToString(h.Sum(nil)) + + pack.Index.HashFormat = "sha256" + pack.Index.Hash = hashString + return nil +} + +// Write saves the pack file +func (pack Pack) Write() error { + f, err := os.Create(pack.flags.PackFile) + if err != nil { + return err + } + defer f.Close() + + enc := toml.NewEncoder(f) + // Disable indentation + enc.Indent = "" + return enc.Encode(pack) +} + diff --git a/main.go b/main.go index 1f48a94..a28b0d5 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ func main() { } func cmdDelete(flags core.Flags) error { + // TODO: actual input mod := "demagnetize" err := os.Remove(core.ResolveMod(mod, flags)) if err != nil { @@ -60,7 +61,11 @@ func cmdDelete(flags core.Flags) error { } func cmdRefresh(flags core.Flags) error { - index, err := core.LoadIndex(flags) + pack, err := core.LoadPack(flags) + if err != nil { + return cli.NewExitError(err, 1) + } + index, err := pack.LoadIndex() if err != nil { return cli.NewExitError(err, 1) } @@ -72,6 +77,14 @@ func cmdRefresh(flags core.Flags) error { if err != nil { return cli.NewExitError(err, 1) } + err = pack.UpdateIndexHash() + if err != nil { + return cli.NewExitError(err, 1) + } + err = pack.Write() + if err != nil { + return cli.NewExitError(err, 1) + } return nil }