Revert "Removed spark implementation, use a docker-compose cluster for testing"

This reverts commit 8943eceb93.

Makefile

@@ -9,6 +9,9 @@ UID := $(shell id -u)
 mobileinsight.sif: mobileinsight.def
 	"$(APPTAINER)" build --build-arg="NUM_THREADS=$(NUM_THREADS)" "$@" "$<"
 
+mobileinsight-spark.sif: with-spark.def mobileinsight.sif
+	"$(APPTAINER)" build "$@" "$<"
+
 prepare:
 	@mkdir -p "$(OVERLAY_DIR)"
 	@lsmod | grep overlay > /dev/null || sudo modprobe overlay
@@ -17,6 +20,10 @@ run: mobileinsight.sif prepare
 	@$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \
 		-B "/run/user/$(UID)" mobileinsight.sif
 
+run-spark: mobileinsight-spark.sif prepare
+	@$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \
+		-B "/run/user/$(UID)" mobileinsight-spark.sif
+
 clean:
 	rm -rf *.sif $(OVERLAY_DIR)
 

with-spark.def

@@ -0,0 +1,32 @@
+Bootstrap: localimage
+From: mobileinsight.sif
+
+%arguments
+    SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz"
+
+%environment
+    export SPARK_HOME="/opt/spark"
+    export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH"
+    export PYSPARK_PYTHON="/usr/bin/python3"
+    export SPARK_LOCAL_IP="127.0.0.1"
+
+%post
+    # Update and install dependencies
+    mkdir -p /build
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+    apt-get upgrade -y
+    apt-get install -y \
+        openjdk-8-jre-headless \
+        scala
+
+    # Download Spark
+    SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")"
+    cd /build
+    wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ"
+    tar -xf "$SPARK_TGZ"
+    mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark
+
+    # Cleanup
+    apt-get autoclean -y
+    rm -rf /build