inkweaver
/
mobileinsight-apptainer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Simplified spark version so pyspark can work

This commit is contained in:
Jack Yu 2024-05-02 19:27:04 -07:00
parent 7300236791
commit 37c4bad792
1 changed files with 1 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
with-spark.def
View File

@ -5,9 +5,6 @@ From: mobileinsight.sif
SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz"
%environment %environment
export SPARK_HOME="/opt/spark"
export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH"
export PYSPARK_PYTHON="/usr/bin/python3"
export SPARK_LOCAL_IP="127.0.0.1" export SPARK_LOCAL_IP="127.0.0.1"
%post %post
@ -19,14 +16,7 @@ From: mobileinsight.sif
apt-get install -y \ apt-get install -y \
openjdk-8-jre-headless \ openjdk-8-jre-headless \
scala scala
pip3 install pyspark==3.5.1
# Download Spark
SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")"
cd /build
wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ"
tar -xf "$SPARK_TGZ"
mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark
# Cleanup # Cleanup
apt-get autoclean -y apt-get autoclean -y
rm -rf /build