Commit 0aeb443e authored by Gmodena's avatar Gmodena
Browse files

Orchestrate build with make

parent 87cc4b3e
...@@ -33,11 +33,7 @@ jobs: ...@@ -33,11 +33,7 @@ jobs:
java-version: adopt@1.8 java-version: adopt@1.8
- name: Install Spark - name: Install Spark
run: | run: |
wget http://apachemirror.wuchna.com/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz make install_spark
tar -xzvf spark-2.4.7-bin-hadoop2.7.tgz
- name: Test with pytest - name: Test with pytest
run: | run: |
export SPARK_HOME=spark-2.4.7-bin-hadoop2.7/ make test
export PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip:$PYTHONPATH
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
pytest tests/
spark_version = 2.4.7
hadoop_version = 2.7
spark_home = spark-${spark_version}-bin-hadoop${hadoop_version}
spark_tgz_url = http://apachemirror.wuchna.com/spark/spark-${spark_version}/${spark_home}.tgz
venv: requirements.txt venv: requirements.txt
test -d venv || virtualenv --python=$(shell which python3) venv test -d venv || python -m venv venv
. venv/bin/activate; pip install -Ur requirements.txt; . venv/bin/activate; pip install -Ur requirements.txt;
test: venv install_spark:
test -d ${spark_version} || wget ${spark_tgz_url}; tar -xzvf ${spark_home}.tgz
clean_spark:
rm -r ${spark_home}
test: install_spark venv
. venv/bin/activate; pytest --cov etl . venv/bin/activate; pytest --cov etl
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment