Commit f9330b0c authored by Gmodena's avatar Gmodena
Browse files

Run CI containers with Wikimedia images.

parent 9a16c404
......@@ -12,12 +12,19 @@
# Stages run in sequential order, but jobs within stages run in parallel.
# For more information, see:
image: "continuumio/miniconda3"
# This includes a wikimedia-buster docker image
# with conda installed.
- project: 'repos/data-engineering/workflow_utils'
ref: 'v0.2.2'
file: '/gitlab_ci_templates/conda.yml'
- conda install -c conda-forge make curl
- conda install python=3.7 openjdk=8.0.152
- pip install tox==3.24.4 cookiecutter==1.7.3
- !reference [.setup_conda, before_script]
- /opt/conda/bin/conda install python=3.7 openjdk=8.0.152
- pip3 install tox==3.24.4
# Parallelize the execution of CI checks (per project).
FROM continuumio/miniconda3
RUN conda install python=3.7 openjdk=8.0.152
RUN pip install tox==3.24.4 cookiecutter==1.7.3
# A Docker file to build a buster image with python 3 and miniconda.
# It's based atop Wikimedia's official python3 image, and its runtime
# matches the container that runs Gitlab CI.
RUN apt update
RUN apt install -y curl gpg
RUN curl | gpg --dearmor > /tmp/conda.gpg
RUN install -o root -g root -m 644 /tmp/conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg
RUN gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806
RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] stable main" > /etc/apt/sources.list.d/conda.list
RUN apt update && apt install -y conda
# OpenJDK 1.8 and Python3.7 are Spark 2.4.x requirements.
RUN bash -c "source /opt/conda/etc/profile.d/; /opt/conda/bin/conda install python=3.7 openjdk=8.0.152"
RUN pip3 install tox==3.24.4 cookiecutter==1.7.3
......@@ -7,11 +7,33 @@ pip_requirements_test := requirements-test.txt
conda_python_version = 3.7
pyspark_version = 2.4.5
CONDA_CMD := conda config --set pip_interop_enabled True; \
conda create -n ${venv} python=${conda_python_version}; \
conda init bash; \
source ~/.bashrc; \
conda activate ${venv}
# Initialize conda's python distribution.
# Conda path on stat nodes
WMF_STAT_CONDA_PATH := /usr/lib/anaconda-wmf
# Conda path on debian buster
DEBIAN_CONDA_PATH := /opt/conda/
# Try to determine if we are running on stat nodes,
# by checking if /usr/lib/anaconda-wmf exists.
# Otherwise default to /opt/conda/.
# This assumes that we are either running on stat nodes, or inside a Docker container.
# Ovverride: if CONDA_PATH is set in the environment, use that path instead.
ifneq ($(CONDA_PATH),)
else ifeq ("$(wildcard $(WMF_STAT_CONDA_PATH))","")
CONDA_INIT := source ${CONDA_PATH}/etc/profile.d/; \
export PATH="${CONDA_PATH}/bin:${PATH}"
# Create a conda environment, that is later packaged for distribution.
# TODO(gmodena): replace this boilerplate with conda-dist
CONDA_CMD := ${CONDA_INIT}; ${CONDA_PATH}/bin/conda config --set pip_interop_enabled True; \
${CONDA_PATH}/bin/conda create -n ${venv} python=${conda_python_version}; \
${CONDA_PATH}/bin/conda activate ${venv}
DOCKER_IMG := platform/miniconda3
DOCKERFILE := Dockerfile.python
......@@ -32,18 +54,19 @@ endif
docker buildx build --platform ${DOCKER_PLATFORM} -t ${DOCKER_IMG} -f ${DOCKERFILE} .
# TODO(gmodena): replace this boilerplate with conda-dist
venv: ${pip_requirements}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements}; \
conda deactivate; \
conda install conda-pack; \
${CONDA_PATH}/bin/conda deactivate; \
${CONDA_PATH}/bin/conda install conda-pack; \
conda-pack -n ${venv} --format ${venv_archive_format}"
mypy: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e mypy"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e mypy"
lint: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e flake8"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e flake8"
test: ${pip_requirements_test}
${DOCKER_CMD} bash -c "tox -e pytest"
${DOCKER_CMD} bash -c "${CONDA_INIT}; tox -e pytest"
......@@ -110,4 +110,8 @@ Python]( distribution. They can be run "nativel
make test SKIP_DOCKER=true
On wikimedia stat machines we expect conda to be installed under `/usr/lib/anaconda-wmf`. The default conda path
can be overridden by passing `CONDA_PATH` to `make`. For example:
make test SKIP_DOCKER=true CONDA_PATH=/path/to/conda
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment