Commit 90ec8494 authored by Gmodena's avatar Gmodena
Browse files

Merge branch 'T292741-implement-ci-checks' into 'multi-project-dags-repo'

T292741 implement ci checks

See merge request gmodena/platform-airflow-dags!8
parents 17187136 6b9d5be4
name: build
on: [push]
runs-on: ubuntu-latest
continue-on-error: true
max-parallel: 4
python-version: [3.7, ]
pet-data-pipeline: ["image-matching", ]
- uses: actions/checkout@v2
- uses: conda-incubator/setup-miniconda@v2
activate-environment: venv
python-version: ${{ matrix.python-version }}
auto-activate-base: false
- name: Lint with flake8
run: cd ${{ }}; make lint SKIP_DOCKER=true
- name: Type check with mypy
run: cd ${{ }}; make mypy SKIP_DOCKER=true
- name: Test with pytest
run: cd ${{ }}; make test SKIP_DOCKER=true
image: "continuumio/miniconda3"
- build
- test
- publish
# TODO(gmodena): cross-project artifacts download
# are not available in Gitlab Freemium.
# stage: build-ima
# script:
# - ls -lhR
# needs:
# - project: gmodena/ImageMatching
# job: build
# ref: add-gitlab-ci
# artifacts: true
# curl is not installed by default on continuumio/miniconda3.
- conda install -y make curl
# Install IMA dependencies in a virtual environment
# and publish it as a build artifact.
# TODO(gmodena): venvs produced by conda-pack and venv-pack
# exceed the artifact size limit provided by gitlab.
# Currently env and dep building is delegated to the "deploy"
# host. In our case, the developer workstation where
# `make deploy` is executed.
# stage: build
# script:
# - make -C image-matching venv
# artifacts:
# paths:
# - image-matching/venv.tar.gz
# expire_in: "1 hour"
# Install Apache Spark in the test image.
# TODO(gmodena): we should have a polyglot Docker image with jdk8, python3,
# and spark pre-installed
stage: test
- make test SKIP_DOCKER=true
stage: publish
- make archive SKIP_DOCKER=true
- 'curl -v --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file /tmp/platform-airflow-dags.tar.gz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/platform-airflow-dags/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/platform-airflow-dags.tar.gz"'
......@@ -27,6 +27,9 @@ ima-venv:
rm -f ${ima_home}/${ima_venv_archive}
make -C ${ima_home} venv
cd ${ima_home}; make lint
test_dags: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
pip install -r ${pip_requirements_test}; \
# wmf-platform-airflow-dags
Experiments with airflow repo and code structure.
......@@ -7,6 +7,20 @@ mypy: ${pip_requirements_test}
pip install -r ${pip_requirements_test}; \
mypy spark"
# TODO(gmodena, 2021-11-01): this conflicts with Makefile layout changes in
lint: ${pip_requirements_test}
# check for syntax errors or undefined names in ${lint_targets} files.
# exit-zero treats all errors as warnings.
# the GitHub editor is 127 chars wide; set that as ax-line-length.
${DOCKER_CMD} bash -c "pip install -r ${pip_requirements_test}; ${CONDA_CMD}; \
flake8 ${lint_targets} \
--count \
--max-complexity=10 \
--max-line-length=80 \
--select=E9,F63,F7,F82 \
--show-source \
test: ${pip_requirements_test}
${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
conda install openjdk pyspark==${pyspark_version}; \
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment