diff --git a/with-spark.def b/with-spark.def index 101bed4..21aea93 100644 --- a/with-spark.def +++ b/with-spark.def @@ -5,9 +5,6 @@ From: mobileinsight.sif SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" %environment - export SPARK_HOME="/opt/spark" - export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH" - export PYSPARK_PYTHON="/usr/bin/python3" export SPARK_LOCAL_IP="127.0.0.1" %post @@ -19,14 +16,7 @@ From: mobileinsight.sif apt-get install -y \ openjdk-8-jre-headless \ scala - - # Download Spark - SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")" - cd /build - wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ" - tar -xf "$SPARK_TGZ" - mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark + pip3 install pyspark==3.5.1 # Cleanup apt-get autoclean -y - rm -rf /build