Commit 2b03bb25 authored by Gmodena's avatar Gmodena Committed by GitHub
Browse files

Update spark dep in makefile (#28)

* Update spark dep in makefile

* Spark version bump in build.yml
parent 3c782c71
......@@ -28,11 +28,11 @@ jobs:
java-version: adopt@1.8
- name: Install Apache Spark
run: |
# This command will install vanilla spark under ./spark-2.4.7-bin-hadoop2.7
# This command will install vanilla spark under ./spark-2.4.8-bin-hadoop2.7
make install_spark
- name: Test with pytest
run: |
export SPARK_HOME=$(pwd)/spark-2.4.7-bin-hadoop2.7
export SPARK_HOME=$(pwd)/spark-2.4.8-bin-hadoop2.7
export PYTHONPATH=${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.7-src.zip:${PYTHONPATH}
export PATH=${PATH}:${SPARK_HOME}/bin:${SPARK_HOME}/sbin
make test
spark_version := 2.4.7
spark_version := 2.4.8
hadoop_version := 2.7
spark_home := spark-${spark_version}-bin-hadoop${hadoop_version}
spark_tgz_url := http://apachemirror.wuchna.com/spark/spark-${spark_version}/${spark_home}.tgz
spark_tgz_url := https://downloads.apache.org/spark/spark-${spark_version}/${spark_home}.tgz
venv: requirements.txt
test -d venv || python3 -m venv venv
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment