Merge branch 'T292741-implement-ci-checks' into 'multi-project-dags-repo'

T292741 implement ci checks

See merge request !8

.github/workflows/build.yml

+name: build
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.7, ]
+        pet-data-pipeline: ["image-matching", ]
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+          activate-environment: venv
+          python-version:  ${{ matrix.python-version }}
+          auto-activate-base: false
+    - name: Lint with flake8
+      run: cd ${{ matrix.pet-data-pipeline }}; make lint SKIP_DOCKER=true
+    - name: Type check with mypy
+      run: cd ${{ matrix.pet-data-pipeline }}; make mypy SKIP_DOCKER=true
+    - name: Test with pytest
+      run: cd ${{ matrix.pet-data-pipeline }}; make test SKIP_DOCKER=true

.gitlab-ci.yml

-image: "continuumio/miniconda3"
-stages:
-  - build
-  - test
-  - publish
-# TODO(gmodena): cross-project artifacts download
-# are not available in Gitlab Freemium.
-#build-ima-upstream:
-#  stage: build-ima
-#  script:
-#    - ls -lhR
-#  needs:
-#    - project: gmodena/ImageMatching
-#      job: build
-#      ref: add-gitlab-ci
-#      artifacts: true
-
-# curl is not installed by default on continuumio/miniconda3.
-before_script:
-  - conda install -y make curl
-
-# Install IMA dependencies in a virtual environment
-# and publish it as a build artifact.
-# TODO(gmodena): venvs produced by conda-pack and venv-pack
-# exceed the artifact size limit provided by gitlab.
-# Currently env and dep building is delegated to the "deploy"
-# host. In our case, the developer workstation where
-# `make deploy` is executed.
-#build-ima-deps:
-#  stage: build
-#  script:
-#    - make -C image-matching venv
-#  artifacts:
-#    paths:
-#      - image-matching/venv.tar.gz
-#    expire_in: "1 hour"
-
-# Install Apache Spark in the test image.
-# TODO(gmodena): we should have a polyglot Docker image with jdk8, python3,
-# and spark pre-installed
-test-ima:
-  stage: test
-  script:
-    - make test SKIP_DOCKER=true
-
-publish-dags:
-  stage: publish
-  script:
-    - make archive SKIP_DOCKER=true
-    - 'curl -v --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file /tmp/platform-airflow-dags.tar.gz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/platform-airflow-dags/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/platform-airflow-dags.tar.gz"'

Makefile

@@ -27,6 +27,9 @@ ima-venv:
 	rm -f ${ima_home}/${ima_venv_archive}
 	make -C ${ima_home} venv
 
+lint:
+	cd ${ima_home}; make lint
+
 test_dags: ${pip_requirements_test}
 	${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
 		pip install -r ${pip_requirements_test}; \

README.md

+[![build](https://github.com/gmodena/wmf-platform-airflow-dags/actions/workflows/build.yml/badge.svg)](https://github.com/gmodena/wmf-platform-airflow-dags/actions/workflows/build.yml)
+
 # wmf-platform-airflow-dags
 
 Experiments with airflow repo and code structure.

image-matching/Makefile

@@ -7,6 +7,20 @@ mypy: ${pip_requirements_test}
     					pip install -r ${pip_requirements_test}; \
     					mypy spark"
 
+# TODO(gmodena, 2021-11-01): this conflicts with Makefile layout changes in https://gitlab.wikimedia.org/gmodena/platform-airflow-dags/-/tree/T293382-add-typing
+lint: ${pip_requirements_test}
+	# check for syntax errors or undefined names in ${lint_targets} files.
+	# exit-zero treats all errors as warnings.
+	# the GitHub editor is 127 chars wide; set that as ax-line-length.
+	${DOCKER_CMD} bash -c "pip install -r ${pip_requirements_test}; ${CONDA_CMD}; \
+			flake8 ${lint_targets} \
+			--count \
+			--max-complexity=10 \
+			--max-line-length=80 \
+			--select=E9,F63,F7,F82 \
+			--show-source \
+			--statistics"
+
 test:   ${pip_requirements_test}
 	${DOCKER_CMD} bash -c "export CONDA_ALWAYS_YES=true; ${CONDA_CMD}; \
 					conda install openjdk pyspark==${pyspark_version}; \