Merge branch 'ci-demo-cleanup' into 'multi-project-dags-repo'

Ci demo cleanup

See merge request gmodena/platform-airflow-dags!10

.github/workflows/build.yml

@@ -3,8 +3,7 @@ name: build
 on: [push]
 
 jobs:
-  build:
-
+  codechecks:
     runs-on: ubuntu-latest
     continue-on-error: true
     strategy:
@@ -28,3 +27,18 @@ jobs:
     - name: Test with pytest
       if: always()
       run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true
+  dag-validation:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+    - name: Install tox
+      run: pip install tox
+    - name: Test with pytest
+      if: always()
+      run: make test_dags SKIP_DOCKER=true
+

Dockerfile

-FROM continuumio/miniconda3
-RUN conda install python=3.7
-RUN pip install tox

Makefile

-include Makefile.conda
+include Makefile.python
 
 branch := $(shell git rev-parse --abbrev-ref HEAD)
 short_commit_hash := $(shell git rev-parse --short=8 HEAD)
@@ -16,6 +16,10 @@ platform_airflow_dags_url := ${gitlab_ci_api_root}/projects/${gitlab_project_id}
 ima_home := image-matching
 ima_venv_archive := venv.tar.gz
 
+ifneq ($(SKIP_DOCKER),true)
+test_dags: docker-conda
+endif
+
 # Runs some command to setup DAGs, venvs and project code on an airflow worker.
 install-dags:
 	ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
@@ -31,9 +35,7 @@ lint:
 	cd ${ima_home}; make lint
 
 test_dags: ${pip_requirements_test}
-	${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
-		pip install -r ${pip_requirements_test}; \
-		python -m pytest tests/"
+	${DOCKER_CMD} bash -c "tox -e dags" 
 
 test:
 	cd  ${ima_home}; make mypy; make test

Makefile.python

@@ -8,14 +8,18 @@ conda_python_version = 3.7
 pyspark_version = 2.4.5
 extra_pypi := https://gitlab.wikimedia.org/api/v4/projects/40/packages/pypi/simple
 
-CONDA_CMD := conda config --set pip_interop_enabled True; conda create -n ${venv} python=${conda_python_version}; conda init bash; source ~/.bashrc && conda activate ${venv}
+CONDA_CMD := conda config --set pip_interop_enabled True; \
+	conda create -n ${venv} python=${conda_python_version}; \
+	conda init bash; \
+	source ~/.bashrc; \
+	conda activate ${venv}
 
 DOCKER_IMG := platform/miniconda3
 DOCKERFILE := ../Dockerfile.conda
 
 ifneq ($(SKIP_DOCKER),true)
-    CURRENT_DIR := $(shell pwd)
-	DOCKER_CMD := docker run -it \
+CURRENT_DIR := $(shell pwd)
+DOCKER_CMD := docker run -it \
 		--rm \
 		-v ${CURRENT_DIR}:/root \
 		-e SKIP_DOCKER=true \

README.md

@@ -7,9 +7,9 @@
 
 This repo contains data pipelines operationalised by the Generated Datasets Platform team.
 You can reach out to us at
-* <Add wikitech url>
-* <Add irc channel?>
-* Slack: `#data-platform-value-stream`.
+* TODO: Add wikitech url
+* TODO: Add irc channel
+* Slack: [#data-platform-value-stream](https://wikimedia.slack.com/archives/C02BB8L2S5R).
 
 # Data pipelines
 > […] a pipeline, also known as a data pipeline, is a set of data processing elements connected in series, where the output of one element is the input of the next one. The elements of a pipeline are often executed in parallel or in time-sliced fashion. […] > https://en.wikipedia.org/wiki/Pipeline_(computing)

tox.ini

+[tox]
+envlist = dags
+skipsdist  = True
+
+[testenv]
+deps=
+   -rrequirements-test.txt
+setenv =
+    PYTHONPATH = spark
+commands=
+    python -m pytest tests/
+