Commit f9330b0c authored by Gmodena's avatar Gmodena
Browse files

Run CI containers with Wikimedia images.

parent 9a16c404
......@@ -12,12 +12,19 @@
# Stages run in sequential order, but jobs within stages run in parallel.
#
# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
image: "continuumio/miniconda3"
# This includes a wikimedia-buster docker image
# with conda installed.
include:
- project: 'repos/data-engineering/workflow_utils'
ref: 'v0.2.2'
file: '/gitlab_ci_templates/conda.yml'
before_script:
- conda install -c conda-forge make curl
- conda install python=3.7 openjdk=8.0.152
- pip install tox==3.24.4 cookiecutter==1.7.3
- !reference [.setup_conda, before_script]
- /opt/conda/bin/conda install python=3.7 openjdk=8.0.152
- pip3 install tox==3.24.4
# Parallelize the execution of CI checks (per project).
.build_matrix:
......
FROM continuumio/miniconda3
RUN conda install python=3.7 openjdk=8.0.152
RUN pip install tox==3.24.4 cookiecutter==1.7.3
# A Docker file to build a buster image with python 3 and miniconda.
# It's based atop Wikimedia's official python3 image, and its runtime
# matches the container that runs Gitlab CI.
FROM docker-registry.wikimedia.org/python3-build-buster:0.1.0-20220313
RUN apt update
RUN apt install -y curl gpg
RUN curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > /tmp/conda.gpg
RUN install -o root -g root -m 644 /tmp/conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg
RUN gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806
RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list
RUN apt update && apt install -y conda
# OpenJDK 1.8 and Python3.7 are Spark 2.4.x requirements.
RUN bash -c "source /opt/conda/etc/profile.d/conda.sh; /opt/conda/bin/conda install python=3.7 openjdk=8.0.152"
RUN pip3 install tox==3.24.4 cookiecutter==1.7.3
......@@ -7,11 +7,33 @@ pip_requirements_test := requirements-test.txt
conda_python_version = 3.7
pyspark_version = 2.4.5
CONDA_CMD := conda config --set pip_interop_enabled True; \
conda create -n ${venv} python=${conda_python_version}; \
conda init bash; \
source ~/.bashrc; \
conda activate ${venv}
# Initialize conda's python distribution.
# Conda path on stat nodes
WMF_STAT_CONDA_PATH := /usr/lib/anaconda-wmf
# Conda path on debian buster
DEBIAN_CONDA_PATH := /opt/conda/
# Try to determine if we are running on stat nodes,
# by checking if /usr/lib/anaconda-wmf exists.
# Otherwise default to /opt/conda/.
# This assumes that we are either running on stat nodes, or inside a Docker container.
# Ovverride: if CONDA_PATH is set in the environment, use that path instead.
ifneq ($(CONDA_PATH),)
CONDA_PATH := ${CONDA_PATH}
else ifeq ("$(wildcard $(WMF_STAT_CONDA_PATH))","")
CONDA_PATH := ${DEBIAN_CONDA_PATH}
else
CONDA_PATH := ${WMF_STAT_CONDA_PATH}
endif
CONDA_INIT := source ${CONDA_PATH}/etc/profile.d/conda.sh; \
export PATH="${CONDA_PATH}/bin:${PATH}"
# Create a conda environment, that is later packaged for distribution.
# TODO(gmodena): replace this boilerplate with conda-dist https://phabricator.wikimedia.org/T303839
CONDA_CMD := ${CONDA_INIT}; ${CONDA_PATH}/bin/conda config --set pip_interop_enabled True; \
${CONDA_PATH}/bin/conda create -n ${venv} python=${conda_python_version}; \
${CONDA_PATH}/bin/conda activate ${venv}
DOCKER_IMG := platform/miniconda3
DOCKERFILE := Dockerfile.python
......@@ -32,18 +54,19 @@ endif
docker-conda:
docker buildx build --platform ${DOCKER_PLATFORM} -t ${DOCKER_IMG} -f ${DOCKERFILE} .
# TODO(gmodena): replace this boilerplate with conda-dist https://phabricator.wikimedia.org/T303839
venv: ${pip_requirements}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements}; \
conda deactivate; \
conda install conda-pack; \
${CONDA_PATH}/bin/conda deactivate; \
${CONDA_PATH}/bin/conda install conda-pack; \
conda-pack -n ${venv} --format ${venv_archive_format}"
mypy: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e mypy"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e mypy"
lint: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e flake8"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e flake8"
test: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e pytest"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e pytest"
......@@ -110,4 +110,8 @@ Python](https://docs.conda.io/en/latest/) distribution. They can be run "nativel
```
make test SKIP_DOCKER=true
```
On wikimedia stat machines we expect conda to be installed under `/usr/lib/anaconda-wmf`. The default conda path
can be overridden by passing `CONDA_PATH` to `make`. For example:
```
make test SKIP_DOCKER=true CONDA_PATH=/path/to/conda
```
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment