Commit 8bdf986a authored by Addshore's avatar Addshore 🏄
Browse files

Refactor embed files -> disk as util

parent 0db74e6b
/*Package files for interacting packaged files and their counterparts on disk for a project directory
Copyright © 2020 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package files
import (
"os"
"path/filepath"
)
/*EnsureReady makes sure that the files component is ready.*/
func EnsureReady(projectDirectory string) {
ensureInMemoryFilesAreOnDisk(projectDirectory)
}
/*ListRawDcYamlFilesInContextOfProjectDirectory ...*/
func ListRawDcYamlFilesInContextOfProjectDirectory(projectDirectory string) []string {
// TODO this function should live in the mwdd struct?
var files []string
for _, file := range listRawFiles(projectDirectory) {
if filepath.Ext(file) == ".yml" {
files = append(files, filepath.Base(file))
}
}
return files
}
/*listRawFiles lists the raw docker-compose file paths that are currently on disk.*/
func listRawFiles(projectDirectory string) []string {
var files []string
err := filepath.Walk(projectDirectory, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, path)
}
return nil
})
if err != nil {
panic(err)
}
return files
}
/*Package files for interacting packaged files and their counterparts on disk for a project directory
Copyright © 2020 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package files
import (
"bytes"
"embed"
"fmt"
"io"
"io/fs"
"io/ioutil"
"strings"
"os"
"path/filepath"
"gitlab.wikimedia.org/releng/cli/internal/util/embedsync"
)
//go:embed embed
var content embed.FS
func strippedFileName(name string) string {
return strings.TrimPrefix(name, "embed/")
func syncer(projectDirectory string) embedsync.EmbeddingDiskSync {
return embedsync.EmbeddingDiskSync{
Embed: content,
EmbedPath: "embed",
DiskPath: projectDirectory,
}
}
func files() []string {
return replaceInAllStrings(strings.Split(strings.Trim(indexString(), "\n"), "\n"), "./", "embed/")
/*EnsureReady makes sure that the files component is ready.*/
func EnsureReady(projectDirectory string) {
syncer := syncer(projectDirectory)
syncer.EnsureFilesOnDisk()
}
func fileBytes(name string) []byte {
fileReader := fileReaderOrExit(name)
bytes, _ := ioutil.ReadAll(fileReader)
return bytes
}
/*ListRawDcYamlFilesInContextOfProjectDirectory ...*/
func ListRawDcYamlFilesInContextOfProjectDirectory(projectDirectory string) []string {
// TODO this function should live in the mwdd struct?
var files []string
func fileString(name string) string {
fileReader := fileReaderOrExit(name)
buf := bytes.NewBuffer(nil)
io.Copy(buf, fileReader)
fileReader.Close()
return buf.String()
for _, file := range listRawFiles(projectDirectory) {
if filepath.Ext(file) == ".yml" {
files = append(files, filepath.Base(file))
}
}
return files
}
func fileReaderOrExit(name string) fs.File {
fileReader, err := content.Open(name)
/*listRawFiles lists the raw docker-compose file paths that are currently on disk.*/
func listRawFiles(projectDirectory string) []string {
var files []string
err := filepath.Walk(projectDirectory, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, path)
}
return nil
})
if err != nil {
fmt.Println("Failed to open file: " + name)
fmt.Println(err)
panic(err)
}
return fileReader
}
func replaceInAllStrings(list []string, find string, replace string) []string {
for i, s := range list {
list[i] = strings.Replace(s, find, replace, -1)
}
return list
}
func indexString() string {
return fileString("embed/files.txt")
return files
}
package files
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
)
/*EnsureInMemoryFilesAreOnDisk makes sure that up to date copies of our in app docker-compose files are on disk
TODO this should be called only when we update the bin?
TODO This is way to complex, and should be more clever.. checking a hash or something? and be more lightweight.*/
func ensureInMemoryFilesAreOnDisk(projectDirectory string) {
embededFiles := files()
// Ensure each file is on disk and up to date
for _, embedFileName := range embededFiles {
strippedFileName := strippedFileName(embedFileName)
fileTargetOnDisk := projectDirectory + string(os.PathSeparator) + strippedFileName
packagedBytes := fileBytes(embedFileName)
if _, err := os.Stat(fileTargetOnDisk); os.IsNotExist(err) {
// TODO only output the below line with verbose logging
// fmt.Println(fileTargetOnDisk + " doesn't exist, so write it...")
writeBytesToDisk(packagedBytes, fileTargetOnDisk)
} else {
onDiskBytes := diskFileToBytes(fileTargetOnDisk)
if !bytes.Equal(onDiskBytes, packagedBytes) {
// TODO only output the below line with verbose logging
// fmt.Println(fileTargetOnDisk + " out of date, so writing...")
writeBytesToDisk(packagedBytes, fileTargetOnDisk)
}
}
}
}
func diskFileToBytes(file string) []byte {
bytes, _ := ioutil.ReadFile(file)
// TODO check error?
return bytes
}
func getAssumedFilePerms(filePath string) os.FileMode {
// Set all .sh files as +x when creating them
// All users should be able to read and execute these files so users in containers can use them
// XXX: Currently if you change these file permissions on disk files will need to be deleted and re added..
if filepath.Ext(filePath) == ".sh" {
return 0o755
}
return 0o655
}
func writeBytesToDisk(bytes []byte, filePath string) {
ensureDirectoryForFileOnDisk(filePath)
ioutil.WriteFile(filePath, bytes, getAssumedFilePerms(filePath))
// TODO check error?
}
func ensureDirectoryForFileOnDisk(file string) {
ensureDirectoryOnDisk(filepath.Dir(file))
}
func ensureDirectoryOnDisk(dirPath string) {
if _, err := os.Stat(dirPath); err != nil {
os.MkdirAll(dirPath, 0o755)
}
}
package dirs
import "os"
func EnsureExists(dirPath string) {
if _, err := os.Stat(dirPath); err != nil {
mkerr := os.MkdirAll(dirPath, 0o755)
if mkerr != nil {
panic(mkerr)
}
}
}
/*Package embedsync deals with syncing go embeded files onto the systme disk
NOTE: this requires an index of the files to be part of the embed.
This can be generated in the MakeFile using a line like this...
@cd ./internal/mwdd/files/embed/ && find . -type f > files.txt
Copyright © 2021 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package embedsync
import (
"bytes"
"embed"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"strings"
"gitlab.wikimedia.org/releng/cli/internal/util/dirs"
"gitlab.wikimedia.org/releng/cli/internal/util/files"
utilstrings "gitlab.wikimedia.org/releng/cli/internal/util/strings"
)
type EmbeddingDiskSync struct {
Embed embed.FS
EmbedPath string
DiskPath string
}
func (e EmbeddingDiskSync) EnsureFilesOnDisk() {
embededFiles := e.files()
// Ensure each file is on disk and up to date
for _, embedFileName := range embededFiles {
strippedFileName := e.strippedFileName(embedFileName)
fileTargetOnDisk := e.DiskPath + string(os.PathSeparator) + strippedFileName
embededBytes := e.fileBytes(strippedFileName)
if _, err := os.Stat(fileTargetOnDisk); os.IsNotExist(err) {
// TODO only output the below line with verbose logging
// fmt.Println(fileTargetOnDisk + " doesn't exist, so write it...")
writeBytesToDisk(embededBytes, fileTargetOnDisk)
} else {
onDiskBytes := files.Bytes(fileTargetOnDisk)
if !bytes.Equal(onDiskBytes, embededBytes) {
// TODO only output the below line with verbose logging
// fmt.Println(fileTargetOnDisk + " out of date, so writing...")
writeBytesToDisk(embededBytes, fileTargetOnDisk)
}
}
}
}
func (e EmbeddingDiskSync) EnsureNoExtraFilesOnDisk() {
// TODO https://phabricator.wikimedia.org/T282361
panic("not implemented")
}
func (e EmbeddingDiskSync) files() []string {
// "./" switched for EmbedPath as from content of files.txt
return utilstrings.ReplaceInAll(strings.Split(strings.Trim(e.indexString(), "\n"), "\n"), "./", e.EmbedPath+string(os.PathSeparator))
}
func (e EmbeddingDiskSync) indexString() string {
return e.fileString("files.txt")
}
func (e EmbeddingDiskSync) fileString(name string) string {
fileReader := e.fileReaderOrExit(name)
buf := bytes.NewBuffer(nil)
io.Copy(buf, fileReader)
fileReader.Close()
return buf.String()
}
func (e EmbeddingDiskSync) fileBytes(name string) []byte {
fileReader := e.fileReaderOrExit(name)
bytes, _ := ioutil.ReadAll(fileReader)
return bytes
}
func (e EmbeddingDiskSync) fileReaderOrExit(name string) fs.File {
innerName := e.EmbedPath + string(os.PathSeparator) + name
fileReader, err := e.Embed.Open(innerName)
if err != nil {
fmt.Println("Failed to open file: " + innerName)
fmt.Println(err)
panic(err)
}
return fileReader
}
func (e EmbeddingDiskSync) strippedFileName(name string) string {
return strings.TrimPrefix(name, e.EmbedPath+string(os.PathSeparator))
}
func writeBytesToDisk(bytes []byte, file string) {
dirs.EnsureExists(filepath.Dir(file))
ioutil.WriteFile(file, bytes, getAssumedFilePerms(file))
// TODO check error?
}
func getAssumedFilePerms(filePath string) os.FileMode {
// Set all .sh files as +x when creating them
// All users should be able to read and execute these files so users in containers can use them
// XXX: Currently if you change these file permissions on disk files will need to be deleted and re added..
if filepath.Ext(filePath) == ".sh" {
return 0o755
}
return 0o655
}
/*Package embedsync deals with syncing go embeded files onto the systme disk
NOTE: this requires an index of the files to be part of the embed.
This can be generated in the MakeFile using a line like this...
@cd ./internal/mwdd/files/embed/ && find . -type f > files.txt
Copyright © 2021 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package embedsync
import (
"embed"
"io/ioutil"
"log"
"os"
"testing"
)
//go:embed testembed
var testContent embed.FS
func checkFileContent(t *testing.T, file string, expected string) {
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if string(content) != expected {
t.Errorf("Expected %s, got %s", expected, string(content))
}
}
func TestEmbeddingDiskSync_EnsureFilesOnDisk(t *testing.T) {
dir, err := ioutil.TempDir("", "TestEmbeddingDiskSync_EnsureFilesOnDisk")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
e := EmbeddingDiskSync{
Embed: testContent,
EmbedPath: "testembed",
DiskPath: dir,
}
t.Run("Ensure files are actually on disk and look correct", func(t *testing.T) {
e.EnsureFilesOnDisk()
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
// Check count of files
if len(files) != 3 {
t.Errorf("Expected 3 files, got %d", len(files))
}
// Check file contents
checkFileContent(t, dir+"/testfile1", "foo")
checkFileContent(t, dir+"/testfile2.txt", "bar")
checkFileContent(t, dir+"/adir/test3", "hi")
})
t.Run("Ensure files overwritten if changed", func(t *testing.T) {
e.EnsureFilesOnDisk()
file, err := os.Create(dir + "/testfile1")
if err != nil {
t.Fatal(err)
}
defer file.Close()
file.WriteString("changed")
file.Close()
checkFileContent(t, dir+"/testfile1", "changed")
e.EnsureFilesOnDisk()
checkFileContent(t, dir+"/testfile1", "foo")
})
}
./testfile1
./testfile2.txt
./adir/test3
\ No newline at end of file
foo
\ No newline at end of file
......@@ -20,6 +20,7 @@ package files
import (
"bufio"
"bytes"
"io/ioutil"
"os"
"strings"
)
......@@ -69,3 +70,12 @@ func Lines(fileName string) []string {
}
return lines
}
/*Bytes gets bytes of a file or panics.*/
func Bytes(fileName string) []byte {
bytes, err := ioutil.ReadFile(fileName)
if err != nil {
panic(err)
}
return bytes
}
package strings
import "strings"
func ReplaceInAll(list []string, find string, replace string) []string {
for i, s := range list {
list[i] = strings.Replace(s, find, replace, -1)
}
return list
}
package strings
import (
"reflect"
"testing"
)
func TestReplaceInAll(t *testing.T) {
type args struct {
list []string
find string
replace string
}
tests := []struct {
name string
args args
want []string
}{
{
name: "replace in all simple",
args: args{
list: []string{"a", "b", "c"},
find: "a",
replace: "x",
},
want: []string{"x", "b", "c"},
},
{
name: "replace in all complex",
args: args{
list: []string{"abc", "123", "aaa"},
find: "a",
replace: "x",
},
want: []string{"xbc", "123", "xxx"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ReplaceInAll(tt.args.list, tt.args.find, tt.args.replace); !reflect.DeepEqual(got, tt.want) {
t.Errorf("ReplaceInAll() = %v, want %v", got, tt.want)
}
})
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment