Merge branch 'fix-archive-and-deploy' into 'multi-project-dags-repo'

Fix tar archive generation.

See merge request gmodena/platform-airflow-dags!15

Makefile

@@ -24,7 +24,9 @@ endif
 
 # Runs some command to setup DAGs, venvs and project code on an airflow worker.
 install-dags:
-	ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
+	ssh ${airflow_host} 'sudo -u ${airflow_user} rm -r ${airflow_home}/dags/*'
+	ssh ${airflow_host} 'sudo -u ${airflow_user} rm -r ${airflow_home}/image-matching'
+	ssh ${airflow_host} 'sudo -u ${airflow_user} tar xzf ${gitlab_package_archive} -C ${airflow_home}'
 	ssh ${airflow_host} 'sudo -u ${airflow_user} mkdir ${airflow_home}/image-matching/venv'
 	ssh ${airflow_host} 'sudo -u ${airflow_user} tar xvzf ${airflow_home}/image-matching/venv.tar.gz -C ${airflow_home}/image-matching/venv'
 	ssh ${airflow_host} 'rm ${gitlab_package_archive}'
@@ -33,6 +35,8 @@ ima-venv:
 	rm -f ${ima_home}/${ima_venv_archive}
 	cd ${ima_home}; make venv
 
+## Code checks
+
 lint-all:
 	cd ${ima_home}; make lint
 
@@ -46,8 +50,10 @@ test_dags:
 test-all:
 	cd ${ima_home}; make test
 
+## Package dags and project dependencies.
+
 archive: ima-venv
-	tar cvz --exclude=".*" -f ${gitlab_package_archive} .
+	tar cvz --exclude='.[^/]*' --exclude='__pycache__' --exclude='venv/'  -f platform-airflow-dags.tar.gz *
 
 # Publish an artifact to a Gitlab Generic Archive registry using a private token.
 publish: archive
@@ -55,6 +61,14 @@ publish: archive
 	test "x$(status)" = "x" || echo "Echo Working directory is dirty. Aborting."
 	#curl -v --header "PRIVATE-TOKEN: ${GITLAB_PRIVATE_TOKEN}" --upload-file /tmp/platform-airflow-dags.tar.gz "${gitlab_ci_api_root}/projects/${gitlab_project_id}/packages/generic/platform-airflow-dags/${branch}-${short_commit_hash}/${gitlab_package_archive}"
 
+## Deployment
+
+# The following commands automate copying files from a local machine to 
+# a dev airflow instance.
+#
+# They are stubs, meant to be replaced by scap (deploy-giltlab-build) or DE dev tooling
+# (deploy-local-build).
+#
 # Test, assemble venvs, generate an archive locally and ship it to the airflow worker.
 deploy-local-build: test-all archive
 	scp ${gitlab_package_archive} ${airflow_host}: