From 37c4bad792f9ed8c09e8b5d93417ae6dc27f76c0 Mon Sep 17 00:00:00 2001
From: Jack Yu <yiyaoyu@cs.ucla.edu>
Date: Thu, 2 May 2024 19:27:04 -0700
Subject: [PATCH] Simplified spark version so pyspark can work

---
 with-spark.def | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/with-spark.def b/with-spark.def
index 101bed4..21aea93 100644
--- a/with-spark.def
+++ b/with-spark.def
@@ -5,9 +5,6 @@ From: mobileinsight.sif
     SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz"
 
 %environment
-    export SPARK_HOME="/opt/spark"
-    export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH"
-    export PYSPARK_PYTHON="/usr/bin/python3"
     export SPARK_LOCAL_IP="127.0.0.1"
 
 %post
@@ -19,14 +16,7 @@ From: mobileinsight.sif
     apt-get install -y \
         openjdk-8-jre-headless \
         scala
-
-    # Download Spark
-    SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")"
-    cd /build
-    wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ"
-    tar -xf "$SPARK_TGZ"
-    mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark
+    pip3 install pyspark==3.5.1
 
     # Cleanup
     apt-get autoclean -y
-    rm -rf /build