Commit d666bf63 authored by Gmodena's avatar Gmodena
Browse files

Merge branch 'ci-demo-cleanup' into 'multi-project-dags-repo'

Ci demo cleanup

See merge request gmodena/platform-airflow-dags!10
parents f144ae45 03402ed4
...@@ -3,8 +3,7 @@ name: build ...@@ -3,8 +3,7 @@ name: build
on: [push] on: [push]
jobs: jobs:
build: codechecks:
runs-on: ubuntu-latest runs-on: ubuntu-latest
continue-on-error: true continue-on-error: true
strategy: strategy:
...@@ -28,3 +27,18 @@ jobs: ...@@ -28,3 +27,18 @@ jobs:
- name: Test with pytest - name: Test with pytest
if: always() if: always()
run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true
dag-validation:
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.7"
- name: Install tox
run: pip install tox
- name: Test with pytest
if: always()
run: make test_dags SKIP_DOCKER=true
FROM continuumio/miniconda3
RUN conda install python=3.7
RUN pip install tox
include Makefile.conda include Makefile.python
branch := $(shell git rev-parse --abbrev-ref HEAD) branch := $(shell git rev-parse --abbrev-ref HEAD)
short_commit_hash := $(shell git rev-parse --short=8 HEAD) short_commit_hash := $(shell git rev-parse --short=8 HEAD)
...@@ -16,6 +16,10 @@ platform_airflow_dags_url := ${gitlab_ci_api_root}/projects/${gitlab_project_id} ...@@ -16,6 +16,10 @@ platform_airflow_dags_url := ${gitlab_ci_api_root}/projects/${gitlab_project_id}
ima_home := image-matching ima_home := image-matching
ima_venv_archive := venv.tar.gz ima_venv_archive := venv.tar.gz
ifneq ($(SKIP_DOCKER),true)
test_dags: docker-conda
endif
# Runs some command to setup DAGs, venvs and project code on an airflow worker. # Runs some command to setup DAGs, venvs and project code on an airflow worker.
install-dags: install-dags:
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv' ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
...@@ -31,9 +35,7 @@ lint: ...@@ -31,9 +35,7 @@ lint:
cd ${ima_home}; make lint cd ${ima_home}; make lint
test_dags: ${pip_requirements_test} test_dags: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \ ${DOCKER_CMD} bash -c "tox -e dags"
pip install -r ${pip_requirements_test}; \
python -m pytest tests/"
test: test:
cd ${ima_home}; make mypy; make test cd ${ima_home}; make mypy; make test
......
...@@ -8,14 +8,18 @@ conda_python_version = 3.7 ...@@ -8,14 +8,18 @@ conda_python_version = 3.7
pyspark_version = 2.4.5 pyspark_version = 2.4.5
extra_pypi := https://gitlab.wikimedia.org/api/v4/projects/40/packages/pypi/simple extra_pypi := https://gitlab.wikimedia.org/api/v4/projects/40/packages/pypi/simple
CONDA_CMD := conda config --set pip_interop_enabled True; conda create -n ${venv} python=${conda_python_version}; conda init bash; source ~/.bashrc && conda activate ${venv} CONDA_CMD := conda config --set pip_interop_enabled True; \
conda create -n ${venv} python=${conda_python_version}; \
conda init bash; \
source ~/.bashrc; \
conda activate ${venv}
DOCKER_IMG := platform/miniconda3 DOCKER_IMG := platform/miniconda3
DOCKERFILE := ../Dockerfile.conda DOCKERFILE := ../Dockerfile.conda
ifneq ($(SKIP_DOCKER),true) ifneq ($(SKIP_DOCKER),true)
CURRENT_DIR := $(shell pwd) CURRENT_DIR := $(shell pwd)
DOCKER_CMD := docker run -it \ DOCKER_CMD := docker run -it \
--rm \ --rm \
-v ${CURRENT_DIR}:/root \ -v ${CURRENT_DIR}:/root \
-e SKIP_DOCKER=true \ -e SKIP_DOCKER=true \
......
...@@ -7,9 +7,9 @@ ...@@ -7,9 +7,9 @@
This repo contains data pipelines operationalised by the Generated Datasets Platform team. This repo contains data pipelines operationalised by the Generated Datasets Platform team.
You can reach out to us at You can reach out to us at
* <Add wikitech url> * TODO: Add wikitech url
* <Add irc channel?> * TODO: Add irc channel
* Slack: `#data-platform-value-stream`. * Slack: [#data-platform-value-stream](https://wikimedia.slack.com/archives/C02BB8L2S5R).
# Data pipelines # Data pipelines
> […] a pipeline, also known as a data pipeline, is a set of data processing elements connected in series, where the output of one element is the input of the next one. The elements of a pipeline are often executed in parallel or in time-sliced fashion. […] > https://en.wikipedia.org/wiki/Pipeline_(computing) > […] a pipeline, also known as a data pipeline, is a set of data processing elements connected in series, where the output of one element is the input of the next one. The elements of a pipeline are often executed in parallel or in time-sliced fashion. […] > https://en.wikipedia.org/wiki/Pipeline_(computing)
......
[tox]
envlist = dags
skipsdist = True
[testenv]
deps=
-rrequirements-test.txt
setenv =
PYTHONPATH = spark
commands=
python -m pytest tests/
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment