Commit 79075dee authored by Gmodena's avatar Gmodena
Browse files

Run dag validation in CI

parent f144ae45
......@@ -3,8 +3,7 @@ name: build
on: [push]
jobs:
build:
codechecks:
runs-on: ubuntu-latest
continue-on-error: true
strategy:
......@@ -28,3 +27,19 @@ jobs:
- name: Test with pytest
if: always()
run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true
dag-validation:
runs-on: ubuntu-latest
continue-on-error: true
python-version: ["3.7", ]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install tox
run: pip install tox
- name: Test with pytest
if: always()
run: make test_dags SKIP_DOCKER=true
include Makefile.conda
include Makefile.python
branch := $(shell git rev-parse --abbrev-ref HEAD)
short_commit_hash := $(shell git rev-parse --short=8 HEAD)
......@@ -16,6 +16,10 @@ platform_airflow_dags_url := ${gitlab_ci_api_root}/projects/${gitlab_project_id}
ima_home := image-matching
ima_venv_archive := venv.tar.gz
ifneq ($(SKIP_DOCKER),true)
test_dags: docker-conda
endif
# Runs some command to setup DAGs, venvs and project code on an airflow worker.
install-dags:
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
......@@ -31,9 +35,7 @@ lint:
cd ${ima_home}; make lint
test_dags: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements_test}; \
python -m pytest tests/"
${DOCKER_CMD} bash -c "tox -e dags"
test:
cd ${ima_home}; make mypy; make test
......
......@@ -8,14 +8,18 @@ conda_python_version = 3.7
pyspark_version = 2.4.5
extra_pypi := https://gitlab.wikimedia.org/api/v4/projects/40/packages/pypi/simple
CONDA_CMD := conda config --set pip_interop_enabled True; conda create -n ${venv} python=${conda_python_version}; conda init bash; source ~/.bashrc && conda activate ${venv}
CONDA_CMD := conda config --set pip_interop_enabled True; \
conda create -n ${venv} python=${conda_python_version}; \
conda init bash; \
source ~/.bashrc; \
conda activate ${venv}
DOCKER_IMG := platform/miniconda3
DOCKERFILE := ../Dockerfile.conda
ifneq ($(SKIP_DOCKER),true)
CURRENT_DIR := $(shell pwd)
DOCKER_CMD := docker run -it \
CURRENT_DIR := $(shell pwd)
DOCKER_CMD := docker run -it \
--rm \
-v ${CURRENT_DIR}:/root \
-e SKIP_DOCKER=true \
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment