🚧 This instance is under construction; expect occasional downtime. Runners available in /repos. Questions? Ask in #wikimedia-gitlab on libera.chat, or under GitLab on Phabricator.

Commit 6b9d5be4 authored by Gmodena's avatar Gmodena
Browse files

T292741 implement ci checks

parent 17187136
name: build
on: [push]
jobs:
build:
runs-on: ubuntu-latest
continue-on-error: true
strategy:
max-parallel: 4
matrix:
python-version: [3.7, ]
pet-data-pipeline: ["image-matching", ]
steps:
- uses: actions/checkout@v2
- uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: venv
python-version: ${{ matrix.python-version }}
auto-activate-base: false
- name: Lint with flake8
run: cd ${{ matrix.pet-data-pipeline }}; make lint SKIP_DOCKER=true
- name: Type check with mypy
run: cd ${{ matrix.pet-data-pipeline }}; make mypy SKIP_DOCKER=true
- name: Test with pytest
run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true
image: "continuumio/miniconda3"
stages:
- build
- test
- publish
# TODO(gmodena): cross-project artifacts download
# are not available in Gitlab Freemium.
#build-ima-upstream:
# stage: build-ima
# script:
# - ls -lhR
# needs:
# - project: gmodena/ImageMatching
# job: build
# ref: add-gitlab-ci
# artifacts: true
# curl is not installed by default on continuumio/miniconda3.
before_script:
- conda install -y make curl
# Install IMA dependencies in a virtual environment
# and publish it as a build artifact.
# TODO(gmodena): venvs produced by conda-pack and venv-pack
# exceed the artifact size limit provided by gitlab.
# Currently env and dep building is delegated to the "deploy"
# host. In our case, the developer workstation where
# `make deploy` is executed.
#build-ima-deps:
# stage: build
# script:
# - make -C image-matching venv
# artifacts:
# paths:
# - image-matching/venv.tar.gz
# expire_in: "1 hour"
# Install Apache Spark in the test image.
# TODO(gmodena): we should have a polyglot Docker image with jdk8, python3,
# and spark pre-installed
test-ima:
stage: test
script:
- make test SKIP_DOCKER=true
publish-dags:
stage: publish
script:
- make archive SKIP_DOCKER=true
- 'curl -v --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file /tmp/platform-airflow-dags.tar.gz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/platform-airflow-dags/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/platform-airflow-dags.tar.gz"'
......@@ -27,6 +27,9 @@ ima-venv:
rm -f ${ima_home}/${ima_venv_archive}
make -C ${ima_home} venv
lint:
cd ${ima_home}; make lint
test_dags: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements_test}; \
......
[![build](https://github.com/gmodena/wmf-platform-airflow-dags/actions/workflows/build.yml/badge.svg)](https://github.com/gmodena/wmf-platform-airflow-dags/actions/workflows/build.yml)
# wmf-platform-airflow-dags
Experiments with airflow repo and code structure.
......@@ -7,6 +7,20 @@ mypy: ${pip_requirements_test}
pip install -r ${pip_requirements_test}; \
mypy spark"
# TODO(gmodena, 2021-11-01): this conflicts with Makefile layout changes in https://gitlab.wikimedia.org/gmodena/platform-airflow-dags/-/tree/T293382-add-typing
lint: ${pip_requirements_test}
# check for syntax errors or undefined names in ${lint_targets} files.
# exit-zero treats all errors as warnings.
# the GitHub editor is 127 chars wide; set that as ax-line-length.
${DOCKER_CMD} bash -c "pip install -r ${pip_requirements_test}; ${CONDA_CMD}; \
flake8 ${lint_targets} \
--count \
--max-complexity=10 \
--max-line-length=80 \
--select=E9,F63,F7,F82 \
--show-source \
--statistics"
test: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
conda install openjdk pyspark==${pyspark_version}; \
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment