🚧 This instance is under construction; expect occasional downtime. Runners available in /repos. Questions? Ask in #wikimedia-gitlab on libera.chat, or under GitLab on Phabricator.

Commit 9ae2f586 authored by Gmodena's avatar Gmodena
Browse files

Merge branch 'fix-archive-and-deploy' into 'multi-project-dags-repo'

Fix tar archive generation.

See merge request !15
parents 86bf3428 d08b3333
......@@ -24,7 +24,9 @@ endif
# Runs some command to setup DAGs, venvs and project code on an airflow worker.
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -rf ${airflow_home}/image-matching/venv'
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -r ${airflow_home}/dags/*'
ssh ${airflow_host} 'sudo -u ${airflow_user} rm -r ${airflow_home}/image-matching'
ssh ${airflow_host} 'sudo -u ${airflow_user} tar xzf ${gitlab_package_archive} -C ${airflow_home}'
ssh ${airflow_host} 'sudo -u ${airflow_user} mkdir ${airflow_home}/image-matching/venv'
ssh ${airflow_host} 'sudo -u ${airflow_user} tar xvzf ${airflow_home}/image-matching/venv.tar.gz -C ${airflow_home}/image-matching/venv'
ssh ${airflow_host} 'rm ${gitlab_package_archive}'
......@@ -33,6 +35,8 @@ ima-venv:
rm -f ${ima_home}/${ima_venv_archive}
cd ${ima_home}; make venv
## Code checks
cd ${ima_home}; make lint
......@@ -46,8 +50,10 @@ test_dags:
cd ${ima_home}; make test
## Package dags and project dependencies.
archive: ima-venv
tar cvz --exclude=".*" -f ${gitlab_package_archive} .
tar cvz --exclude='.[^/]*' --exclude='__pycache__' --exclude='venv/' -f platform-airflow-dags.tar.gz *
# Publish an artifact to a Gitlab Generic Archive registry using a private token.
publish: archive
......@@ -55,6 +61,14 @@ publish: archive
test "x$(status)" = "x" || echo "Echo Working directory is dirty. Aborting."
#curl -v --header "PRIVATE-TOKEN: ${GITLAB_PRIVATE_TOKEN}" --upload-file /tmp/platform-airflow-dags.tar.gz "${gitlab_ci_api_root}/projects/${gitlab_project_id}/packages/generic/platform-airflow-dags/${branch}-${short_commit_hash}/${gitlab_package_archive}"
## Deployment
# The following commands automate copying files from a local machine to
# a dev airflow instance.
# They are stubs, meant to be replaced by scap (deploy-giltlab-build) or DE dev tooling
# (deploy-local-build).
# Test, assemble venvs, generate an archive locally and ship it to the airflow worker.
deploy-local-build: test-all archive
scp ${gitlab_package_archive} ${airflow_host}:
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment