Commit 7c64770a authored by Addshore's avatar Addshore 🏄
Browse files

fresh support in dev command

parent f789c60f
......@@ -105,7 +105,7 @@ integration-docker:
- TEST: docker-mw-mysql-suspend-resume-destroy.sh
before_script:
# libc6-compat needed because https://stackoverflow.com/questions/36279253/go-compiled-binary-wont-run-in-an-alpine-docker-container-on-ubuntu-host
- apk add --no-cache libc6-compat bash docker-compose curl composer
- apk add --no-cache libc6-compat bash docker-compose curl
- ./tests/cache-mediawiki.sh
- ./tests/setup.sh
script:
......
......@@ -519,9 +519,11 @@ var applyRelevantWorkingDirectory = func(dockerExecCommand mwdd.DockerExecComman
// For paths inside the mediawiki path, rewrite things
if strings.HasPrefix(currentWorkingDirectory, mountedMwDirectory) {
dockerExecCommand.WorkingDir = strings.Replace(currentWorkingDirectory, mountedMwDirectory, "/var/www/html/w", 1)
} else {
// Otherwise just use the root of mediawiki
dockerExecCommand.WorkingDir = "/var/www/html/w"
}
// Otherwise just use the root of mediawiki
return dockerExecCommand
}
......
/*Package cmd is used for command line.
Copyright © 2020 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cmd
import (
"os"
"github.com/spf13/cobra"
"gitlab.wikimedia.org/releng/cli/internal/exec"
"gitlab.wikimedia.org/releng/cli/internal/mwdd"
)
var mwddMediawikiFreshCmd = &cobra.Command{
Use: "fresh ...",
Short: "Runs commands in a 'fresh' container.",
Long: `Runs commands in a 'fresh' container.
Various environment variables are already set up for you.
- MW_SERVER=http://default.mediawiki.mwdd:${PORT}
- MW_SCRIPT_PATH=/w
- MEDIAWIKI_USER=Admin
- MEDIAWIKI_PASSWORD=mwddpassword
Note: the lack of .localhost at the end of the site name. Using .localhost will NOT work in this container.`,
Example: ` # Start an interactive terminal in the fresh container
fresh bash
# Run npm ci in the currently directory (if within mediawiki)
fresh npm ci # Run npm ci in the current directory (if within mediawiki)
# Run mediawiki core tests (when in the mediawiki core directory)
fresh npm run selenium-test
# Run a single Wikibase extension test spec (when in the Wikibase extension directory)
fresh npm run selenium-test:repo -- -- --spec repo/tests/selenium/specs/item.js`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 {
cmd.Help()
os.Exit(1)
}
mwdd.DefaultForUser().EnsureReady()
options := exec.HandlerOptions{
Verbosity: Verbosity,
}
mwdd.DefaultForUser().UpDetached([]string{"mediawiki-fresh"}, options)
mwdd.DefaultForUser().DockerRun(applyRelevantWorkingDirectory(mwdd.DockerExecCommand{
DockerComposeService: "mediawiki-fresh",
Command: args,
User: User,
}))
},
}
func init() {
mwddMediawikiCmd.AddCommand(mwddMediawikiFreshCmd)
mwddMediawikiFreshCmd.Flags().StringVarP(&User, "user", "u", mwdd.UserAndGroupForDockerExecution(), "User to run as, defaults to current OS user uid:gid")
}
......@@ -622,6 +622,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
......@@ -961,6 +962,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
......@@ -972,6 +974,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
......@@ -1237,6 +1240,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
......@@ -1419,6 +1423,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
......
......@@ -28,6 +28,7 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/signal"
"gitlab.wikimedia.org/releng/cli/internal/exec"
......@@ -57,6 +58,7 @@ func UserAndGroupForDockerExecution() string {
func (m MWDD) DockerExec(command DockerExecCommand) {
containerID := m.DockerComposeProjectName() + "_" + command.DockerComposeService + "_1"
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
fmt.Println("Unable to create docker client")
......@@ -73,7 +75,6 @@ func (m MWDD) DockerExec(command DockerExecCommand) {
Cmd: []string{"/bin/sh", "-c", strings.Join(command.Command, " ")},
}
ctx := context.Background()
response, err := cli.ContainerExecCreate(ctx, containerID, execConfig)
if err != nil {
return
......@@ -127,6 +128,93 @@ func (m MWDD) DockerExec(command DockerExecCommand) {
}
}
/*DockerRun runs a docker container using the docker SDK attached to the mwdd network etc...*/
func (m MWDD) DockerRun(command DockerExecCommand) {
containerID := m.DockerComposeProjectName() + "_" + command.DockerComposeService + "_1"
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
fmt.Println("Unable to create docker client")
panic(err)
}
containerJSON, _ := cli.ContainerInspect(context.Background(), containerID)
containerConfig := containerJSON.Config
containerConfig.AttachStderr = true
containerConfig.AttachStdout = true
containerConfig.AttachStdin = true
containerConfig.OpenStdin = true
containerConfig.Tty = true
containerConfig.WorkingDir = command.WorkingDir
containerConfig.User = command.User
containerConfig.Entrypoint = []string{"/bin/sh"}
containerConfig.Cmd = []string{"-c", strings.Join(command.Command, " ")}
// Remove the old one and start a new one with new options :)
cli.ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{Force: true})
resp, err := cli.ContainerCreate(
ctx,
containerConfig,
containerJSON.HostConfig,
&network.NetworkingConfig{
EndpointsConfig: containerJSON.NetworkSettings.Networks,
},
nil,
containerID,
)
if err != nil {
panic(err)
}
waiter, err := cli.ContainerAttach(ctx, resp.ID, types.ContainerAttachOptions{
Stream: true,
Stdin: containerConfig.AttachStdin,
Stdout: containerConfig.AttachStdout,
Stderr: containerConfig.AttachStderr,
})
if err != nil {
fmt.Println(err)
return
}
if containerConfig.Tty {
if err := monitorTtySize(ctx, cli, resp.ID, false); err != nil {
fmt.Println("Error monitoring TTY size:")
fmt.Println(err)
}
}
cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
// When TTY is ON, just copy stdout https://phabricator.wikimedia.org/T282340
// See: https://github.com/docker/cli/blob/70a00157f161b109be77cd4f30ce0662bfe8cc32/cli/command/container/hijack.go#L121-L130
go io.Copy(os.Stdout, waiter.Reader)
go io.Copy(waiter.Conn, os.Stdin)
fd := int(os.Stdin.Fd())
var oldState *terminal.State
if terminal.IsTerminal(fd) {
oldState, _ = terminal.MakeRaw(fd)
defer terminal.Restore(fd, oldState)
}
for {
resp, err := cli.ContainerInspect(ctx, resp.ID)
time.Sleep(50 * time.Millisecond)
if err != nil {
break
}
if !resp.State.Running {
break
}
}
}
// MonitorTtySize updates the container tty size when the terminal tty changes size
func monitorTtySize(ctx context.Context, client client.APIClient, id string, isExec bool) error {
// Source: https://github.com/skiffos/skiff-core/blob/82c430e4961453c250883c2e5ebd4bd360fa13a5/shell/tty.go
......
......@@ -4,12 +4,12 @@ services:
adminer:
image: "${ADMINER_IMAGE:-adminer:4}"
environment:
- VIRTUAL_HOST=adminer.mwdd.localhost
- VIRTUAL_HOST=adminer.mwdd.localhost,adminer.mwdd
- VIRTUAL_PORT=8080
depends_on:
- dps
- nginx-proxy
hostname: adminer.mwdd.localhost
hostname: adminer.mwdd
dns:
- 10.0.0.10
networks:
......
......@@ -15,18 +15,18 @@ services:
# TODO: replace with jwilder/nginx-proxy, once updated
image: silvanwmde/nginx-proxy@sha256:4488c32bbe8c2ec5b806c38af5f591f5dafe0aa835591a2285017ca59bd12a75
environment:
- VIRTUAL_HOST=proxy.mwdd.localhost
- HOSTNAMES=.mediawiki.mwdd.localhost # wildcard name resolution, thanks to DPS
- VIRTUAL_HOST=proxy.mwdd.localhost,proxy.mwdd
- HOSTNAMES=.mediawiki.mwdd.localhost,.mediawiki.mwdd # wildcard name resolution, thanks to DPS
- HTTP_PORT=${PORT} # internal port
ports:
- "${PORT}:${PORT}"
depends_on:
- dps
hostname: proxy.mwdd.localhost
hostname: proxy.mwdd
dns:
- 10.0.0.10
dns_search:
- mwdd.localhost
- mwdd
networks:
- dps
volumes:
......
......@@ -4,8 +4,8 @@ services:
graphite:
image: graphiteapp/graphite-statsd:1.1.8-1
environment:
- VIRTUAL_HOST=graphite.mwdd.localhost
hostname: graphite.mwdd.localhost
- VIRTUAL_HOST=graphite.mwdd.localhost,graphite.mwdd
hostname: graphite.mwdd
depends_on:
- nginx-proxy
dns:
......
version: '3.7'
# Service wrapper around what is provided by fresh
# https://github.com/wikimedia/fresh/blob/master/bin/fresh-node10
services:
mediawiki-fresh:
image: docker-registry.wikimedia.org/releng/node14-test-browser:0.0.2
entrypoint: /bin/sh
command: -c "echo started"
working_dir: /var/www/html/w
networks:
- dps
dns:
- 10.0.0.10
environment:
# https://www.mediawiki.org/wiki/Selenium/How-to/Run_tests_targeting_MediaWiki-Docker_using_Fresh#Environment_variables
- MW_SERVER=http://default.mediawiki.mwdd:${PORT}
- MW_SCRIPT_PATH=/w
- MEDIAWIKI_USER=Admin
- MEDIAWIKI_PASSWORD=mwddpassword
volumes:
# Only mount code and config, don't mount logs or image
# TODO is this bit of config even needed?
- "${MEDIAWIKI_VOLUMES_CODE}:/var/www/html/w:cached"
- ./mediawiki:/mwdd:ro
\ No newline at end of file
......@@ -20,13 +20,13 @@ services:
- COMPOSER_CACHE_DIR=/.composer/cache
- XDEBUG_CONFIG=${MEDIAWIKI_XDEBUG_CONFIG:-}
- XDEBUG_MODE=${MEDIAWIKI_XDEBUG_MODE:-develop,debug}
hostname: mediawiki
hostname: mediawiki.mwdd
depends_on:
- mediawiki-web
dns:
- 10.0.0.10
dns_search:
- mwdd.localhost
- mwdd
networks:
- dps
......@@ -37,9 +37,9 @@ services:
- "${MEDIAWIKI_VOLUMES_DATA:-mediawiki-data}:/var/www/html/w/data:delegated"
- "${MEDIAWIKI_VOLUMES_IMAGES:-mediawiki-images}:/var/www/html/w/images/docker:delegated"
environment:
- VIRTUAL_HOST=*.mediawiki.mwdd.localhost
- VIRTUAL_HOST=*.mediawiki.mwdd.localhost,*.mediawiki.mwdd
- VIRTUAL_PORT=8080
hostname: mediawiki-web.mwdd.localhost
hostname: mediawiki-web.mwdd
depends_on:
- nginx-proxy
dns:
......
......@@ -6,7 +6,7 @@ services:
image: "${MYSQL_IMAGE:-mariadb:10.5}"
environment:
- MYSQL_ROOT_PASSWORD=toor
hostname: mysql-replica.mwdd.localhost
hostname: mysql-replica.mwdd
depends_on:
- mysql
- mysql-replica-configure-replication
......
......@@ -5,7 +5,7 @@ services:
image: "${MYSQL_IMAGE:-mariadb:10.5}"
environment:
- MYSQL_ROOT_PASSWORD=toor
hostname: mysql.mwdd.localhost
hostname: mysql.mwdd
depends_on:
- mysql-configure-replication
dns:
......
......@@ -8,11 +8,11 @@ services:
- PMA_PASSWORD=toor
- PMA_HOSTS=mysql,mysql-replica
- PMA_ARBITRARY=1
- VIRTUAL_HOST=phpmyadmin.mwdd.localhost
- VIRTUAL_HOST=phpmyadmin.mwdd.localhost,phpmyadmin.mwdd
depends_on:
- dps
- nginx-proxy
hostname: phpmyadmin.mwdd.localhost
hostname: phpmyadmin.mwdd
dns:
- 10.0.0.10
networks:
......
......@@ -7,7 +7,7 @@ services:
# Specify "root" rather than the default "postgres" so that it is similar to the mysql service
- POSTGRES_USER=root
- POSTGRES_PASSWORD=toor
hostname: postgres.mwdd.localhost
hostname: postgres.mwdd
dns:
- 10.0.0.10
networks:
......
......@@ -3,7 +3,7 @@ version: '3.7'
services:
redis:
image: "${REDIS_IMAGE:-redis:6.2}"
hostname: redis.mwdd.localhost
hostname: redis.mwdd
dns:
- 10.0.0.10
networks:
......
......@@ -4,7 +4,7 @@ set -e # Fail on errors
set -x # Output commands
# Only re fetch MediaWiki if we don't already have it in the cache for this job
if [[ ! -f mediawiki/.gitlab-ci.cache.20210809-02 ]]; then
if [[ ! -f mediawiki/.gitlab-ci.cache.20210809-04 ]]; then
rm -rf mediawiki
apk add --no-cache curl tar
curl https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/core/+archive/refs/heads/master.tar.gz -o mediawiki.tar.gz
......@@ -17,12 +17,19 @@ if [[ ! -f mediawiki/.gitlab-ci.cache.20210809-02 ]]; then
mkdir mediawiki/vendor
tar -xf vendor.tar.gz -C mediawiki/vendor
rm -r *.tar.gz
# composer install
docker run --rm -v $PWD/mediawiki:/app -w /app --entrypoint=composer docker-registry.wikimedia.org/dev/stretch-php73-fpm:3.0.0 install --ignore-platform-reqs --no-interaction
touch mediawiki/.gitlab-ci.cache.20210809-02
# npm install
apk add --no-cache npm
npm --prefix mediawiki ci
touch mediawiki/.gitlab-ci.cache.20210809-04
fi
# Always do a composer update (even when cached) to ensure deps are as up to date as possible
composer update --working-dir=mediawiki --no-interaction --no-progress --ignore-platform-reqs
# composer update (even when cached) to ensure deps are as up to date as possible
docker run --rm -v $PWD/mediawiki:/app -w /app --entrypoint=composer docker-registry.wikimedia.org/dev/stretch-php73-fpm:3.0.0 update --no-interaction --no-progress --ignore-platform-reqs
# Always remove files that may have been left behind by previous tests and may get in the way?
rm -f mediawiki/LocalSettings.php
......@@ -40,11 +40,16 @@ cd mediawiki
# composer: Make sure a command works in root of the repo
./../bin/mw docker mediawiki composer home | grep -q "https://www.mediawiki.org/"
# exec: Make sure a command works in the root of the repo
./../bin/mw docker mediawiki exec ls | grep -q "api.php"
# exec phpunit: Make sure using exec to run phpunit things works
./../bin/mw docker mediawiki exec -- composer phpunit tests/phpunit/unit/includes/PingbackTest.php
./../bin/mw docker mediawiki exec -- composer phpunit tests/phpunit/unit/includes/PingbackTest.php | grep -q "OK "
# exec: Make sure a command works in the root of the repo
./../bin/mw docker mediawiki exec ls | grep -q "api.php"
# fresh: Make sur a basic browser test works
./../bin/mw docker mediawiki fresh npm run selenium-test -- -- --spec tests/selenium/specs/page.js
# cd to Vector
cd skins/Vector
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment