Commit d666bf63 authored by Gmodena's avatar Gmodena
Browse files

Merge branch 'ci-demo-cleanup' into 'multi-project-dags-repo'

Ci demo cleanup

See merge request gmodena/platform-airflow-dags!10
parents f144ae45 03402ed4
......@@ -3,8 +3,7 @@ name: build
on: [push]
jobs:
build:
codechecks:
runs-on: ubuntu-latest
continue-on-error: true
strategy:
......@@ -28,3 +27,18 @@ jobs:
- name: Test with pytest
if: always()
run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true
dag-validation:
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.7"
- name: Install tox
run: pip install tox
- name: Test with pytest
if: always()
run: make test_dags SKIP_DOCKER=true
FROM continuumio/miniconda3
RUN conda install python=3.7
RUN pip install tox
include Makefile.conda
include Makefile.python
branch := $(shell git rev-parse --abbrev-ref HEAD)
short_commit_hash := $(shell git rev-parse --short=8 HEAD)
......@@ -16,6 +16,10 @@ platform_airflow_dags_url := ${gitlab_ci_api_root}/projects/${gitlab_project_id}
ima_home := image-matching
ima_venv_archive := venv.tar.gz
ifneq ($(SKIP_DOCKER),true)
test_dags: docker-conda
endif
# Runs some command to setup DAGs, venvs and project code on an airflow worker.
install-dags:
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
......@@ -31,9 +35,7 @@ lint:
cd ${ima_home}; make lint
test_dags: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements_test}; \
python -m pytest tests/"
${DOCKER_CMD} bash -c "tox -e dags"
test:
cd ${ima_home}; make mypy; make test
......
......@@ -8,14 +8,18 @@ conda_python_version = 3.7
pyspark_version = 2.4.5
extra_pypi := https://gitlab.wikimedia.org/api/v4/projects/40/packages/pypi/simple
CONDA_CMD := conda config --set pip_interop_enabled True; conda create -n ${venv} python=${conda_python_version}; conda init bash; source ~/.bashrc && conda activate ${venv}
CONDA_CMD := conda config --set pip_interop_enabled True; \
conda create -n ${venv} python=${conda_python_version}; \
conda init bash; \
source ~/.bashrc; \
conda activate ${venv}
DOCKER_IMG := platform/miniconda3
DOCKERFILE := ../Dockerfile.conda
ifneq ($(SKIP_DOCKER),true)
CURRENT_DIR := $(shell pwd)
DOCKER_CMD := docker run -it \
CURRENT_DIR := $(shell pwd)
DOCKER_CMD := docker run -it \
--rm \
-v ${CURRENT_DIR}:/root \
-e SKIP_DOCKER=true \
......
......@@ -7,9 +7,9 @@
This repo contains data pipelines operationalised by the Generated Datasets Platform team.
You can reach out to us at
* <Add wikitech url>
* <Add irc channel?>
* Slack: `#data-platform-value-stream`.
* TODO: Add wikitech url
* TODO: Add irc channel
* Slack: [#data-platform-value-stream](https://wikimedia.slack.com/archives/C02BB8L2S5R).
# Data pipelines
> […] a pipeline, also known as a data pipeline, is a set of data processing elements connected in series, where the output of one element is the input of the next one. The elements of a pipeline are often executed in parallel or in time-sliced fashion. […] > https://en.wikipedia.org/wiki/Pipeline_(computing)
......
[tox]
envlist = dags
skipsdist = True
[testenv]
deps=
-rrequirements-test.txt
setenv =
PYTHONPATH = spark
commands=
python -m pytest tests/
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment