Commit 55f6c00e authored by Thcipriani's avatar Thcipriani 💬
Browse files

Merge branch 'master' into debian

Change-Id: I5c2360255aca8e4a96efc909a12c617411675c8d
parents cabf89de af9e7979
version: v2
version: v3
base: golang:1.9-stretch
lives: { in: /go/src/gerrit.wikimedia.org/r/blubber }
variants:
test:
runs: { insecurely: true }
builder: [go, get, -u, github.com/golang/lint/golint]
builder:
command: [go, get, -u, github.com/golang/lint/golint]
entrypoint: [make, test]
SHELL := /bin/bash
RELEASE_DIR ?= ./_release
TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64le windows/amd64 plan9/amd64
......@@ -15,18 +16,18 @@ install:
# workaround bug in case CURDIR is a symlink
# see https://github.com/golang/go/issues/24359
cd "$(REAL_CURDIR)" && \
go install -v -ldflags "$(GO_LDFLAGS)"
go install -v -ldflags "$(GO_LDFLAGS)" $(GO_PACKAGES)
release:
gox -output="$(RELEASE_DIR)/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' -ldflags '$(GO_LDFLAGS)' $(PACKAGE)
gox -output="$(RELEASE_DIR)/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' -ldflags '$(GO_LDFLAGS)' $(GO_PACKAGES)
cp LICENSE "$(RELEASE_DIR)"
for f in "$(RELEASE_DIR)"/*/blubber; do \
for f in "$(RELEASE_DIR)"/*/{blubber,blubberoid}; do \
shasum -a 256 "$${f}" | awk '{print $$1}' > "$${f}.sha256"; \
done
lint:
@echo > .lint-gofmt.diff
@go list -f $(GO_LIST_GOFILES) ./... | while read f; do \
@go list -f $(GO_LIST_GOFILES) $(GO_PACKAGES) | while read f; do \
gofmt -e -d "$${f}" >> .lint-gofmt.diff; \
done
@test -z "$(grep '[^[:blank:]]' .lint-gofmt.diff)" || (echo "gofmt found errors:"; cat .lint-gofmt.diff; exit 1)
......
......@@ -31,7 +31,6 @@ variants:
development:
includes: [build]
sharedvolume: true
test:
includes: [build]
......
......@@ -24,7 +24,6 @@ variants:
development:
includes: [build]
sharedvolume: true
test:
includes: [build]
......
......@@ -173,19 +173,6 @@ func (user User) Compile() []string {
return []string{quote(user.Name)}
}
// Volume is a concrete build instruction for defining a volume mount point
// within the container.
//
type Volume struct {
Path string // volume/mount path
}
// Compile returns the quoted volume path.
//
func (vol Volume) Compile() []string {
return []string{quote(vol.Path)}
}
// WorkingDirectory is a build instruction for defining the working directory
// for future command and entrypoint instructions.
//
......
......@@ -104,12 +104,6 @@ func TestUser(t *testing.T) {
assert.Equal(t, []string{`"foo"`}, i.Compile())
}
func TestVolume(t *testing.T) {
i := build.Volume{"/foo/dir"}
assert.Equal(t, []string{`"/foo/dir"`}, i.Compile())
}
func TestWorkingDirectory(t *testing.T) {
i := build.WorkingDirectory{"/foo/path"}
......
......@@ -17,9 +17,9 @@ import (
const parameters = "config.yaml variant"
var (
showHelp *bool = getopt.BoolLong("help", 'h', "show help/usage")
showVersion *bool = getopt.BoolLong("version", 'v', "show version information")
policyURI *string = getopt.StringLong("policy", 'p', "", "policy file URI", "uri")
showHelp = getopt.BoolLong("help", 'h', "show help/usage")
showVersion = getopt.BoolLong("version", 'v', "show version information")
policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri")
)
func main() {
......
// Package main provides the blubberoid server.
//
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"github.com/pborman/getopt/v2"
"gerrit.wikimedia.org/r/blubber/config"
"gerrit.wikimedia.org/r/blubber/docker"
)
var (
showHelp = getopt.BoolLong("help", 'h', "show help/usage")
address = getopt.StringLong("address", 'a', ":8748", "socket address/port to listen on (default ':8748')", "address:port")
endpoint = getopt.StringLong("endpoint", 'e', "/", "server endpoint (default '/')", "path")
policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri")
policy *config.Policy
)
func main() {
getopt.Parse()
if *showHelp {
getopt.Usage()
os.Exit(1)
}
if *policyURI != "" {
var err error
policy, err = config.ReadPolicyFromURI(*policyURI)
if err != nil {
log.Fatalf("Error loading policy from %s: %v\n", *policyURI, err)
}
}
// Ensure endpoint is always an absolute path starting and ending with "/"
*endpoint = path.Clean("/" + *endpoint)
if *endpoint != "/" {
*endpoint += "/"
}
log.Printf("listening on %s for requests to %s[variant]\n", *address, *endpoint)
http.HandleFunc(*endpoint, blubberoid)
log.Fatal(http.ListenAndServe(*address, nil))
}
func blubberoid(res http.ResponseWriter, req *http.Request) {
if len(req.URL.Path) <= len(*endpoint) {
res.WriteHeader(http.StatusNotFound)
res.Write(responseBody("request a variant at %s[variant]", *endpoint))
return
}
variant := req.URL.Path[len(*endpoint):]
body, err := ioutil.ReadAll(req.Body)
if err != nil {
res.WriteHeader(http.StatusInternalServerError)
log.Printf("failed to read request body: %s\n", err)
return
}
cfg, err := config.ReadConfig(body)
if err != nil {
if config.IsValidationError(err) {
res.WriteHeader(http.StatusUnprocessableEntity)
res.Write(responseBody(config.HumanizeValidationError(err)))
return
}
res.WriteHeader(http.StatusBadRequest)
res.Write(responseBody(
"Failed to read config YAML from request body. "+
"Was it formatted correctly and encoded as binary data?\nerror: %s",
err.Error(),
))
return
}
if policy != nil {
err = policy.Validate(*cfg)
if err != nil {
res.WriteHeader(http.StatusUnprocessableEntity)
res.Write(responseBody(
"Configuration fails policy check against:\npolicy: %s\nviolation: %v",
*policyURI, err,
))
return
}
}
dockerFile, err := docker.Compile(cfg, variant)
if err != nil {
res.WriteHeader(http.StatusNotFound)
res.Write(responseBody(err.Error()))
return
}
res.Header().Set("Content-Type", "text/plain")
res.Write(dockerFile.Bytes())
}
func responseBody(msg string, a ...interface{}) []byte {
return []byte(fmt.Sprintf(msg+"\n", a...))
}
package main
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestBlubberoid(t *testing.T) {
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/test", strings.NewReader(`---
version: v3
base: foo
variants:
test: {}`))
blubberoid(rec, req)
resp := rec.Result()
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "text/plain", resp.Header.Get("Content-Type"))
assert.Contains(t, string(body), "FROM foo")
assert.Contains(t, string(body), `LABEL blubber.variant="test"`)
}
......@@ -8,15 +8,14 @@ import (
// and each configured variant.
//
type CommonConfig struct {
Base string `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image
Apt AptConfig `yaml:"apt"` // APT related
Node NodeConfig `yaml:"node"` // Node related
Python PythonConfig `yaml:"python"` // Python related
Builder BuilderConfig `yaml:"builder"` // Builder related
Lives LivesConfig `yaml:"lives"` // application owner/dir
Runs RunsConfig `yaml:"runs"` // runtime environment
SharedVolume Flag `yaml:"sharedvolume"` // use volume for app
EntryPoint []string `yaml:"entrypoint"` // entry-point executable
Base string `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image
Apt AptConfig `yaml:"apt"` // APT related
Node NodeConfig `yaml:"node"` // Node related
Python PythonConfig `yaml:"python"` // Python related
Builder BuilderConfig `yaml:"builder"` // Builder related
Lives LivesConfig `yaml:"lives"` // application owner/dir
Runs RunsConfig `yaml:"runs"` // runtime environment
EntryPoint []string `yaml:"entrypoint"` // entry-point executable
}
// Merge takes another CommonConfig and merges its fields this one's.
......@@ -32,7 +31,6 @@ func (cc *CommonConfig) Merge(cc2 CommonConfig) {
cc.Builder.Merge(cc2.Builder)
cc.Lives.Merge(cc2.Lives)
cc.Runs.Merge(cc2.Runs)
cc.SharedVolume.Merge(cc2.SharedVolume)
if len(cc.EntryPoint) < 1 {
cc.EntryPoint = cc2.EntryPoint
......
......@@ -12,7 +12,6 @@ func TestCommonConfigYAML(t *testing.T) {
cfg, err := config.ReadConfig([]byte(`---
version: v3
base: fooimage
sharedvolume: true
entrypoint: ["/bin/foo"]
variants:
build: {}`))
......@@ -20,13 +19,11 @@ func TestCommonConfigYAML(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "fooimage", cfg.Base)
assert.Equal(t, true, cfg.SharedVolume.True)
assert.Equal(t, []string{"/bin/foo"}, cfg.EntryPoint)
variant, err := config.ExpandVariant(cfg, "build")
assert.Equal(t, "fooimage", variant.Base)
assert.Equal(t, true, variant.SharedVolume.True)
assert.Equal(t, []string{"/bin/foo"}, variant.EntryPoint)
}
......
......@@ -13,10 +13,8 @@ func TestFlagMerge(t *testing.T) {
version: v3
base: foo
runs: { insecurely: true }
sharedvolume: false
variants:
development:
sharedvolume: true
runs: { insecurely: false }`))
if assert.NoError(t, err) {
......@@ -24,7 +22,6 @@ func TestFlagMerge(t *testing.T) {
if assert.NoError(t, err) {
assert.False(t, variant.Runs.Insecurely.True)
assert.True(t, variant.SharedVolume.True)
}
}
}
......@@ -2,7 +2,6 @@ package config
import (
"gerrit.wikimedia.org/r/blubber/build"
"path"
)
// NodeConfig holds configuration fields related to the Node environment and
......@@ -27,53 +26,47 @@ func (nc *NodeConfig) Merge(nc2 NodeConfig) {
}
// InstructionsForPhase injects instructions into the build related to Node
// dependency installation and setting of the NODE_ENV, NODE_PATH, and PATH
// environment variables.
// dependency installation and setting of the NODE_ENV.
//
// PhasePreInstall
//
// Installs Node package dependencies declared in requirements files into the
// shared library directory (/opt/lib). Only production related packages are
// install if NodeConfig.Env is set to "production" in which case `npm dedupe`
// is also run. Installing dependencies during the build.PhasePreInstall phase
// allows a compiler implementation (e.g. Docker) to produce cache-efficient
// output so only changes to package.json will invalidate these steps of the
// image build.
// application directory. Only production related packages are install if
// NodeConfig.Env is set to "production" in which case `npm dedupe` is also
// run. Installing dependencies during the build.PhasePreInstall phase allows
// a compiler implementation (e.g. Docker) to produce cache-efficient output
// so only changes to package.json will invalidate these steps of the image
// build.
//
// PhasePostInstall
//
// Injects build.Env instructions for NODE_ENV, NODE_PATH, and PATH, setting
// the environment according to the configuration, ensuring that packages
// installed during build.PhasePreInstall are found by Node, and that any
// installed binaries are found by shells.
// Injects build.Env instructions for NODE_ENV, setting the environment
// according to the configuration.
//
func (nc NodeConfig) InstructionsForPhase(phase build.Phase) []build.Instruction {
switch phase {
case build.PhasePreInstall:
if len(nc.Requirements) > 0 {
npmInstall := build.RunAll{[]build.Run{
{"cd", []string{LocalLibPrefix}},
{"npm install", []string{}},
}}
if nc.Env == "production" {
npmInstall.Runs[1].Arguments = []string{"--production"}
npmInstall.Runs[0].Arguments = []string{"--production"}
npmInstall.Runs = append(npmInstall.Runs,
build.Run{"npm dedupe", []string{}},
)
}
return append(
build.SyncFiles(nc.Requirements, LocalLibPrefix),
build.SyncFiles(nc.Requirements, "."),
npmInstall,
)
}
case build.PhasePostInstall:
if nc.Env != "" || len(nc.Requirements) > 0 {
return []build.Instruction{build.Env{map[string]string{
"NODE_ENV": nc.Env,
"NODE_PATH": path.Join(LocalLibPrefix, "node_modules"),
"PATH": path.Join(LocalLibPrefix, "node_modules", ".bin") + ":${PATH}",
"NODE_ENV": nc.Env,
}}}
}
}
......
......@@ -69,9 +69,8 @@ func TestNodeConfigInstructionsNonProduction(t *testing.T) {
t.Run("PhasePreInstall", func(t *testing.T) {
assert.Equal(t,
[]build.Instruction{
build.Copy{[]string{"package.json"}, "/opt/lib/"},
build.Copy{[]string{"package.json"}, "./"},
build.RunAll{[]build.Run{
{"cd", []string{"/opt/lib"}},
{"npm install", []string{}},
}},
},
......@@ -83,9 +82,7 @@ func TestNodeConfigInstructionsNonProduction(t *testing.T) {
assert.Equal(t,
[]build.Instruction{
build.Env{map[string]string{
"NODE_ENV": "foo",
"NODE_PATH": "/opt/lib/node_modules",
"PATH": "/opt/lib/node_modules/.bin:${PATH}",
"NODE_ENV": "foo",
}},
},
cfg.InstructionsForPhase(build.PhasePostInstall),
......@@ -107,9 +104,8 @@ func TestNodeConfigInstructionsProduction(t *testing.T) {
t.Run("PhasePreInstall", func(t *testing.T) {
assert.Equal(t,
[]build.Instruction{
build.Copy{[]string{"package.json", "package-lock.json"}, "/opt/lib/"},
build.Copy{[]string{"package.json", "package-lock.json"}, "./"},
build.RunAll{[]build.Run{
{"cd", []string{"/opt/lib"}},
{"npm install", []string{"--production"}},
{"npm dedupe", []string{}},
}},
......@@ -122,9 +118,7 @@ func TestNodeConfigInstructionsProduction(t *testing.T) {
assert.Equal(t,
[]build.Instruction{
build.Env{map[string]string{
"NODE_ENV": "production",
"NODE_PATH": "/opt/lib/node_modules",
"PATH": "/opt/lib/node_modules/.bin:${PATH}",
"NODE_ENV": "production",
}},
},
cfg.InstructionsForPhase(build.PhasePostInstall),
......@@ -151,9 +145,7 @@ func TestNodeConfigInstructionsEnvironmentOnly(t *testing.T) {
assert.Equal(t,
[]build.Instruction{
build.Env{map[string]string{
"NODE_ENV": "production",
"NODE_PATH": "/opt/lib/node_modules",
"PATH": "/opt/lib/node_modules/.bin:${PATH}",
"NODE_ENV": "production",
}},
},
cfg.InstructionsForPhase(build.PhasePostInstall),
......
......@@ -23,14 +23,12 @@ func (vc *VariantConfig) Merge(vc2 VariantConfig) {
}
// InstructionsForPhase injects build instructions related to artifact
// copying, volume definition or copying of application files, and all common
// configuration.
// copying, copying of application files, and all common configuration.
//
// PhaseInstall
//
// If VariantConfig.Copies is not set, either copy in application files or
// define a shared volume. Otherwise, delegate to
// ArtifactsConfig.InstructionsForPhase.
// If VariantConfig.Copies is not set, copy in application files. Otherwise,
// delegate to ArtifactsConfig.InstructionsForPhase.
//
func (vc *VariantConfig) InstructionsForPhase(phase build.Phase) []build.Instruction {
instructions := vc.CommonConfig.InstructionsForPhase(phase)
......@@ -53,11 +51,7 @@ func (vc *VariantConfig) InstructionsForPhase(phase build.Phase) []build.Instruc
uid, gid = vc.Lives.UID, vc.Lives.GID
if vc.Copies == "" {
if vc.SharedVolume.True {
instructions = append(instructions, build.Volume{vc.Lives.In})
} else {
instructions = append(instructions, build.Copy{[]string{"."}, "."})
}
instructions = append(instructions, build.Copy{[]string{"."}, "."})
}
case build.PhasePostInstall:
......
......@@ -68,19 +68,6 @@ func TestVariantLoops(t *testing.T) {
func TestVariantConfigInstructions(t *testing.T) {
t.Run("PhaseInstall", func(t *testing.T) {
t.Run("shared volume", func(t *testing.T) {
cfg := config.VariantConfig{}
cfg.Lives.In = "/srv/service"
cfg.SharedVolume.True = true
assert.Equal(t,
[]build.Instruction{
build.Volume{"/srv/service"},
},
cfg.InstructionsForPhase(build.PhaseInstall),
)
})
t.Run("standard source copy", func(t *testing.T) {
cfg := config.VariantConfig{}
cfg.Lives.UID = 123
......
......@@ -51,10 +51,6 @@ func NewInstruction(bi build.Instruction) (Instruction, error) {
case build.User:
i.name = "USER"
case build.Volume:
i.name = "VOLUME"
i.array = true
case build.WorkingDirectory:
i.name = "WORKDIR"
}
......
......@@ -113,16 +113,6 @@ func TestUser(t *testing.T) {
}
}
func TestVolume(t *testing.T) {
i := build.Volume{"/foo/dir"}
di, err := docker.NewInstruction(i)
if assert.NoError(t, err) {
assert.Equal(t, "VOLUME [\"/foo/dir\"]\n", di.Compile())
}
}
func TestWorkingDirectory(t *testing.T) {
i := build.WorkingDirectory{"/foo/dir"}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment