Unverified Commit 6d0c204f authored by addshore's avatar addshore Committed by GitHub
Browse files

Merge pull request #10 from addshore/dev-from-gitlab

Dev from gitlab
parents 48a0d166 442cf5ff
......@@ -97,18 +97,6 @@ jobs:
if: steps.cache-mediawiki-vector.outputs.cache-hit != 'true'
run: composer install --no-progress --ansi --working-dir mediawiki
# Runs the integration test in ./test
integration-test-single-file:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: 1.13
- name: Test
run: ./test
collect-integration-tests:
runs-on: ubuntu-latest
steps:
......
image: docker:19.03.12
variables:
# When you use the dind service, you must instruct Docker to talk with
# the daemon started inside of the service. The daemon is available
# with a network connection instead of the default
# /var/run/docker.sock socket. Docker 19.03 does this automatically
# by setting the DOCKER_HOST in
# https://github.com/docker-library/docker/blob/d45051476babc297257df490d22cbd806f1b11e4/19.03/docker-entrypoint.sh#L23-L29
#
# The 'docker' hostname is the alias of the service container as described at
# https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services.
#
# Specify to Docker where to create the certificates. Docker
# creates them automatically on boot, and creates
# `/certs/client` to share between the service and job
# container, thanks to volume mount from config.toml
DOCKER_TLS_CERTDIR: "/certs"
stages:
- build
- test
- integration
- build-release
# The plan would be for this cache to be reused by all jobs.
# Caches curently end up cached per runner, per job concurrency level and per md5 of path?
# So there are potentially 12 caches that end up needing to be populated right now?
# https://forum.gitlab.com/t/confusion-around-ci-docker-cache-volumes-and-sharing-across-jobs-concurrency/56793
# Docker cache volumes look like this runner-<short-token>-project-<id>-concurrent-<concurrency-id>-cache-<md5-of-path>
cache:
- key: mediawiki
paths:
- mediawiki
services:
- name: docker:19.03.12-dind
# Use a registry mirror to avoid hitting docker hub too much as there is a rate limit of 100 pulls per 6 hours
command: ["--registry-mirror", "https://mirror.gcr.io"]
build:
stage: build
needs: []
cache: {}
image: docker-registry.wikimedia.org/golang:1.13-3
artifacts:
paths:
- bin/
script:
- make
build-release:
stage: build-release
needs: []
cache: {}
image: docker-registry.wikimedia.org/golang:1.13-3
artifacts:
paths:
- _release/
script:
# Ideally make would not be needed, only release? But it is needed to install deps currently?
- make
- make release
test:
stage: test
needs: []
cache: {}
image: docker-registry.wikimedia.org/golang:1.13-3
script:
- go get -u golang.org/x/lint/golint
- make test
integration:
stage: integration
needs: [build]
dependencies:
- build
parallel:
matrix:
- TEST: docker-mw-extra-commands.sh
- TEST: docker-mw-install-all-the-dbs.sh
- TEST: docker-mw-mysql-suspend-resume-destroy.sh
before_script:
# libc6-compat needed because https://stackoverflow.com/questions/36279253/go-compiled-binary-wont-run-in-an-alpine-docker-container-on-ubuntu-host
- apk add --no-cache libc6-compat bash docker-compose curl
- ./tests/cache-mediawiki.sh
- ./tests/setup.sh
script:
- ./tests/$TEST
# Changelog
## 1.0.0 (Work in progress)
...
## [https://github.com/addshore/mwcli/releases/tag/v0.1.0-dev-addshore.20210907.1 v0.1.0-dev-addshore.20210907.1]
* Enable updates from releases.wikimedia.org
* Fix segfaults caused by xdebug and `xdebug.var_display_max_` -1 values. ([phabricator](https://phabricator.wikimedia.org/T288363))
* MediaWiki no longer has `ini_set( 'xdebug.var_display_max_depth', -1 );` set
* MediaWiki no longer has `ini_set( 'xdebug.var_display_max_children', -1 );` set
* MediaWiki no longer has `ini_set( 'xdebug.var_display_max_data', -1 );` set
## [https://github.com/addshore/mwcli/releases/tag/v0.1.0-dev-addshore.20210806.1 v0.1.0-dev-addshore.20210806.1]
* Fix mysql server db check complaining about Countable ([phabricator](https://phabricator.wikimedia.org/T287695))
* Prepare for releases from releases.wikimedia.org
* Take backups of LocalSettings incase they get lost
* Create a user .composer directory if it doesn't exist ([phabricator](https://phabricator.wikimedia.org/T288309))
## [https://github.com/addshore/mwcli/releases/tag/v0.1.0-dev-addshore.20210714.1 v0.1.0-dev-addshore.20210714.1]
* Replace docker command with mwdd functionality
* Introduce a dev alias for use with your main development environment command
* Introduced basic cli configuration and config command
## [https://github.com/addshore/mwcli/releases/tag/v0.1.0-dev-addshore.20210703.1 v0.1.0-dev-addshore.20210703.1]
* Improve updater output
* mwdd
** Removed the confusing mwdd create command
** Implemented mwdd suspend and mwdd resume
** Fix most --user options for most exec commands
** Remove duplicate phpunit command
## [https://github.com/addshore/mwcli/releases/tag/v0.1.0-dev-addshore.20210627.1 v0.1.0-dev-addshore.20210627.1]
[https://github.com/addshore/mwcli/compare/v0.1.0-dev-addshore.20210524.1...v0.1.0-dev-addshore.20210627.1 Commits]
* mwdd: Use docker-compose 3.7 file versions
* mwdd: Use stretch-php72-fpm:3.0.0 image for MediaWiki, which fixed XDebug issues
## v0.1.0-dev-addshore.20210524.1
[https://github.com/addshore/mwcli/compare/v0.1.0-dev-addshore.20210523.2...v0.1.0-dev-addshore.20210524.1 Commits]
* Allow users to choose if they update or not
* Check for new updates daily
* mwdd: Make use of a composer cache
* mwdd: Fix permissions of data and log mounts
* mwdd: Internally use maintenance/checkComposerLockUpToDate.php
* mwdd: Add exec commands for all services
## v0.1.0-dev-addshore.20210523.2
[https://github.com/addshore/mwcli/compare/v0.1.0-dev-addshore.20210523.1...v0.1.0-dev-addshore.20210523.2 Commits]
Initial addshore dev build of most mwdd functionality.
......@@ -24,7 +24,8 @@ import (
)
var mwddDockerComposeCmd = &cobra.Command{
Use: "docker-compose",
Use: "docker-compose",
Aliases: []string{"dc"},
Run: func(cmd *cobra.Command, args []string) {
mwdd.DefaultForUser().EnsureReady()
mwdd.DefaultForUser().DockerComposeTTY(
......
......@@ -34,9 +34,10 @@ import (
)
var mwddMediawikiCmd = &cobra.Command{
Use: "mediawiki",
Short: "MediaWiki service",
RunE: nil,
Use: "mediawiki",
Short: "MediaWiki service",
Aliases: []string{"mw"},
RunE: nil,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
cmd.Parent().Parent().PersistentPreRun(cmd, args)
mwdd := mwdd.DefaultForUser()
......@@ -195,8 +196,9 @@ var DbType string
var DbName string
var mwddMediawikiInstallCmd = &cobra.Command{
Use: "install",
Short: "Installs a new MediaWiki site using install.php",
Use: "install",
Short: "Installs a new MediaWiki site using install.php",
Aliases: []string{"i"},
Run: func(cmd *cobra.Command, args []string) {
mediawiki, _ := mediawiki.ForDirectory(mwdd.DefaultForUser().Env().Get("MEDIAWIKI_VOLUMES_CODE"))
if !mediawiki.LocalSettingsIsPresent() {
......@@ -243,8 +245,9 @@ var mwddMediawikiInstallCmd = &cobra.Command{
composerErr := mwdd.DefaultForUser().ExecNoOutput("mediawiki", []string{
"php", "/var/www/html/w/maintenance/checkComposerLockUpToDate.php",
},
exec.HandlerOptions{})
exec.HandlerOptions{}, User)
if composerErr != nil {
fmt.Println("Composer check failed:", composerErr)
prompt := promptui.Prompt{
IsConfirm: true,
Label: "Composer dependencies are not up to date, do you want to composer install?",
......
......@@ -24,9 +24,10 @@ import (
)
var mwddPhpMyAdminCmd = &cobra.Command{
Use: "phpmyadmin",
Short: "phpMyAdmin service",
RunE: nil,
Use: "phpmyadmin",
Short: "phpMyAdmin service",
Aliases: []string{"ppma"},
RunE: nil,
}
var mwddPhpMyAdminCreateCmd = &cobra.Command{
......
......@@ -47,14 +47,14 @@ var updateCmd = &cobra.Command{
c := config.LoadFromDisk()
fmt.Println("You are on the " + c.UpdateChannel + " channel.")
canUpdate, toUpdateTo := updater.CanUpdate(Version, GitSummary, Verbosity >= 2)
canUpdate, toUpdateToOrMessage := updater.CanUpdate(Version, GitSummary, Verbosity >= 2)
if !canUpdate {
fmt.Println("No update available")
fmt.Println(toUpdateToOrMessage)
os.Exit(0)
}
fmt.Println("New update found: " + toUpdateTo)
fmt.Println("New update found: " + toUpdateToOrMessage)
updatePrompt := promptui.Prompt{
Label: " Do you want to update?",
......
/*Package env for interacting with a .env file
Copyright © 2020 Addshore
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package env
import (
"fmt"
"os"
"sort"
"strings"
)
// Copy of godotenv.Write with modifications to never quote
// https://github.com/joho/godotenv/issues/50#issuecomment-364873528
// https://github.com/moby/moby/issues/12997
func writeOverride(envMap map[string]string, filename string) error {
content, error := marshelOverride(envMap)
if error != nil {
return error
}
file, error := os.Create(filename)
if error != nil {
return error
}
_, err := file.WriteString(content)
return err
}
// Copy of godotenv.Marshel with modifications to never quote
// https://github.com/joho/godotenv/issues/50#issuecomment-364873528
// https://github.com/moby/moby/issues/12997
func marshelOverride(envMap map[string]string) (string, error) {
lines := make([]string, 0, len(envMap))
for k, v := range envMap {
lines = append(lines, fmt.Sprintf(`%s=%s`, k, doubleQuoteEscape(v)))
}
sort.Strings(lines)
return strings.Join(lines, "\n"), nil
}
const doubleQuoteSpecialChars = "\\\n\r\"!$`"
func doubleQuoteEscape(line string) string {
for _, c := range doubleQuoteSpecialChars {
toReplace := "\\" + string(c)
if c == '\n' {
toReplace = `\n`
}
if c == '\r' {
toReplace = `\r`
}
line = strings.Replace(line, string(c), toReplace, -1)
}
return line
}
......@@ -59,7 +59,11 @@ func (f DotFile) read() map[string]string {
}
func (f DotFile) write(envMap map[string]string) {
godotenv.Write(envMap, f.Path())
// Override the regular gotdotenv Write method to avoid adding quotes
// https://github.com/joho/godotenv/issues/50#issuecomment-364873528
// https://github.com/moby/moby/issues/12997
//godotenv.Write(envMap, f.Path())
writeOverride(envMap, f.Path())
}
/*Delete a value from the env*/
......@@ -97,4 +101,4 @@ func (f DotFile) Missing(name string) bool {
/*List all values from the env*/
func (f DotFile) List() map[string]string {
return f.read()
}
}
\ No newline at end of file
......@@ -36,6 +36,15 @@ func DefaultForUser() MWDD {
}
func mwddUserDirectory() string {
// user home dir can not be used in Gitlab CI, must use the project dir instead!
// https://medium.com/@patrick.winters/mounting-volumes-in-sibling-containers-with-gitlab-ci-534e5edc4035
// TODO maybe this should be pushed further up and the whole mwcli dir should be moved?!
_, inGitlabCi := os.LookupEnv("GITLAB_CI")
if inGitlabCi {
ciDir, _ := os.LookupEnv("CI_PROJECT_DIR")
return ciDir + ".mwcli/mwdd"
}
currentUser, _ := user.Current()
projectDirectory := currentUser.HomeDir + string(os.PathSeparator) + ".mwcli/mwdd"
return projectDirectory
......@@ -122,14 +131,14 @@ func (m MWDD) Exec(service string, commandAndArgs []string, options exec.Handler
)
}
/*ExecNoOutput runs `docker-compose exec -T <service> <commandAndArgs>` with not output*/
func (m MWDD) ExecNoOutput(service string, commandAndArgs []string, options exec.HandlerOptions) error {
/*ExecNoOutput runs `docker-compose exec -T <service> <commandAndArgs>` with no output*/
func (m MWDD) ExecNoOutput(service string, commandAndArgs []string, options exec.HandlerOptions, user string) error {
options.HandleStdout = func(stdout bytes.Buffer) {}
options.HandleError = func(stderr bytes.Buffer, err error) {}
return m.DockerCompose(
DockerComposeCommand{
Command: "exec",
CommandArguments: append([]string{"-T", service}, commandAndArgs...),
CommandArguments: append([]string{"-T", "--user", user, service}, commandAndArgs...),
HandlerOptions: options,
},
)
......@@ -206,4 +215,4 @@ func (m MWDD) RmVolumes(dcVolumes []string, options exec.HandlerOptions) {
// TODO execIt?
// TODO run?
// TODO runDetatched?
// TODO logsTail?
// TODO logsTail?
\ No newline at end of file
......@@ -26,7 +26,11 @@ func CanUpdate(currentVersion string, gitSummary string, verboseOutput bool) (bo
c := config.LoadFromDisk()
if c.UpdateChannel == config.UpdateChannelDev {
canUpdate, release := CanUpdateFromAddshore(currentVersion, gitSummary, verboseOutput)
return canUpdate, release.Version.String()
if canUpdate {
return canUpdate, release.Version.String()
}
// When canUpdate is false, we dont have a release to get the version string of
return canUpdate, "Can't currently update"
}
if c.UpdateChannel == config.UpdateChannelStable {
return CanUpdateFromWikimedia(currentVersion, gitSummary, verboseOutput)
......@@ -45,4 +49,4 @@ func Update(currentVersion string, gitSummary string, verboseOutput bool) (bool,
panic("Not yet implemented")
}
panic("Unexpected update channel")
}
}
\ No newline at end of file
......@@ -19,9 +19,9 @@ package updater
import (
"io/ioutil"
"log"
"net/http"
"os"
"runtime"
"strings"
"github.com/blang/semver"
......@@ -34,23 +34,23 @@ func CanUpdateFromWikimedia(currentVersion string, gitSummary string, verboseOut
selfupdate.EnableLog()
}
v, err := semver.Parse(strings.Trim(gitSummary, "v"))
if err != nil {
if verboseOutput {
log.Println("Could not parse git summary version, maybe you are not using a real release?")
}
return false, ""
}
latestRelease := latestWikimediaRelease()
if latestRelease == "404" {
return false, "No Wikimedia releases yet"
}
newVersion, err := semver.Parse(strings.Trim(latestRelease, "v"))
newVersion, newErr := semver.Parse(strings.Trim(latestRelease, "v"))
currentVerion, currentErr := semver.Parse(strings.Trim(gitSummary, "v"))
if newErr != nil {
return false, "Could not remote release version?"
}
if currentErr != nil {
return false, "Could not parse current git summary version '" + gitSummary + "', maybe you are not using a real release? Next release would be " + newVersion.String()
}
return v.Compare(newVersion) == 1, newVersion.String()
return currentVerion.Compare(newVersion) == -1, newVersion.String()
}
func latestWikimediaRelease() string {
......@@ -83,12 +83,12 @@ func UpdateFromWikimedia(currentVersion string, gitSummary string, verboseOutput
selfupdate.EnableLog()
}
canUpdate, newVersion := CanUpdateFromWikimedia(currentVersion, gitSummary, verboseOutput)
canUpdate, newVersionOrMessage := CanUpdateFromWikimedia(currentVersion, gitSummary, verboseOutput)
if !canUpdate {
return false, "No update found"
return false, "No update found: " + newVersionOrMessage
}
assetURL := "https://releases.wikimedia.org/mwcli/" + newVersion + "/mw_v" + newVersion + "_linux_amd64"
assetURL := "https://releases.wikimedia.org/mwcli/" + newVersionOrMessage + "/mw_v" + newVersionOrMessage + "_" + runtime.GOOS + "_" + runtime.GOARCH
cmdPath, err := os.Executable()
if err != nil {
......@@ -100,5 +100,5 @@ func UpdateFromWikimedia(currentVersion string, gitSummary string, verboseOutput
return false, "Binary update failed" + err.Error()
}
return true, "Successfully updated to version " + newVersion
return true, "Successfully updated to version " + newVersionOrMessage
}
......@@ -55,7 +55,14 @@ if( file_exists( $IP . '/data/' . $dockerDb . '.sqlite' ) ) {
} else {
// TODO cache this check somehow so that we don't need a query every time...
try{
$mysqlPdo = new PDO( "mysql:host=mysql;dbname=" . $dockerDb, 'root', 'toor' );
$mysqlPdo = new PDO(
"mysql:host=mysql;dbname=" . $dockerDb,
'root',
'toor',
[
PDO::ATTR_TIMEOUT => 1, // in seconds
]
);
$mysqlCheck = $mysqlPdo->query("SHOW DATABASES LIKE \"" . $dockerDb . "\"");
if($mysqlCheck === false) {
var_dump(json_encode($mysqlPdo->errorInfo()));
......@@ -182,10 +189,6 @@ $wgUploadPath = "{$wgScriptPath}/images/docker/{$dockerDb}";
$dockerLogDirectory = "/var/log/mediawiki";
$wgDebugLogFile = "$dockerLogDirectory/debug.log";
ini_set( 'xdebug.var_display_max_depth', -1 );
ini_set( 'xdebug.var_display_max_children', -1 );
ini_set( 'xdebug.var_display_max_data', -1 );
error_reporting( -1 );
ini_set( 'display_errors', 1 );
$wgShowExceptionDetails = true;
......
#!/usr/bin/env -S -i /bin/bash
set -e
set -u
set -x
BASE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export PATH="$PATH:/usr/local/bin/"
export HOME="$(echo ~)"
INSTALL_DEPS="curl make python3-dev libffi-dev gcc libc-dev cargo"
echo "installing dependencies"
sudo apt-get update
DEBIAN_FRONTEND=noninteractive sudo apt-get -y install $INSTALL_DEPS
# Docker-compose should be installed
if ! command -v docker-compose &> /dev/null
then
echo "installing docker-compose"
curl -L "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
fi
# Golang should be installed
if ! command -v go &> /dev/null
then
echo "installing golang"
pushd /tmp
curl -q -o go1.16.4.linux-amd64.tar.gz https://dl.google.com/go/go1.16.4.linux-amd64.tar.gz
tar -xvf go1.16.4.linux-amd64.tar.gz
mv go /usr/local/bin
export GOROOT=/usr/local/bin/go
export GOPATH=$HOME/go
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
popd
fi
export PATH="$PATH:$HOME/go/bin"
title() {
echo
echo "test: $1..."
}
assert() {
actual="$1"
expected="${2:-}"
printf "\"${actual}\" == \"${expected}\"\n"
test "$actual" = "$expected"
}
#build the cli executable
title "Build mwcli"
{
make
mv $BASE/bin/cli $BASE/bin/mw
}
# # get mediawiki
# title "Clone Mediawiki core"
# {
# git clone https://gerrit.wikimedia.org/r/mediawiki/core --depth 1
# cd core
# }
# title "Create config.json file"
# {
# mkdir ~/.mwcli
# echo '{"dev_mode":"docker"}' > ~/.mwcli/config.json
# }
# title "Setup hosts file"
# {
# sudo echo "127.0.0.1 default.mediawiki.mwdd.localhost" >> /etc/hosts
# sudo echo "127.0.0.1 postgreswiki.mediawiki.mwdd.localhost" >> /etc/hosts
# sudo echo "127.0.0.1 mysqlwiki.mediawiki.mwdd.localhost" >> /etc/hosts
# sudo echo "127.0.0.1 phpmyadmin.mwdd.localhost" >> /etc/hosts
# sudo echo "127.0.0.1 adminer.mwdd.localhost" >> /etc/hosts
# }
# title "Output version"
# {
# $BASE/bin/mw version
# }
# title "Setup & Create"
# {
# $BASE/bin/mw docker env set PORT 8080
# $BASE/bin/mw docker env set MEDIAWIKI_VOLUMES_CODE $(pwd)ore
# $BASE/bin/mw docker mediawiki create
# }
# title "Validate the basic stuff"
# {
# $BASE/bin/mw docker docker-compose ps
# $BASE/bin/mw docker env list
# cat ~/.mwcli/mwdd/default/.env
# curl -s -L -N http://default.mediawiki.mwdd.localhost:8080 | grep -q "The MediaWiki logo"
# }
# title "Add the needed LocalSettings"
# {
# echo "<?php" >> core/LocalSettings.php
# echo "//require_once "$IP/includes/PlatformSettings.php";" >> core/LocalSettings.php
# echo "require_once '/mwdd/MwddSettings.php';" >> core/LocalSettings.php
# }
# title "Turn on all of the services"
# {
# $BASE/bin/mw docker mysql-replica create
# $BASE/bin/mw docker postgres create
# $BASE/bin/mw docker phpmyadmin create
# $BASE/bin/mw docker adminer create
# }
# title "Install everything"
# {
# $BASE/bin/mw docker mediawiki install --dbname mysqlwiki --dbtype mysql
# $BASE/bin/mw docker mediawiki install --dbname postgreswiki --dbtype postgres
# $BASE/bin/mw docker mediawiki install
# }
# title "Check the DB tools"
# {
# CURL=$(curl -s -L -N http://phpmyadmin.mwdd.localhost:8080) && echo $CURL && echo $CURL | grep -q "Open new phpMyAdmin window"
# CURL=$(curl -s -L -N http://adminer.mwdd.localhost:8080) && echo $CURL && echo $CURL | grep -q "Login - Adminer"
# }
# title "And check the installed sites"
# {
# CURL=$(curl -s -L -N http://default.mediawiki.mwdd.localhost:8080) && echo $CURL && echo $CURL | grep -q "MediaWiki has been installed"
# CURL=$(curl -s -L -N http://postgreswiki.mediawiki.mwdd.localhost:8080) && echo $CURL && echo $CURL | grep -q "MediaWiki has been installed"
# CURL=$(curl -s -L -N http://mysqlwiki.mediawiki.mwdd.localhost:8080) && echo $CURL && echo $CURL | grep -q "MediaWiki has been installed"