diff --git a/Makefile b/Makefile index 7c658f3..b941ad1 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ UID := $(shell id -u) mobileinsight.sif: mobileinsight.def "$(APPTAINER)" build --build-arg="NUM_THREADS=$(NUM_THREADS)" "$@" "$<" +mobileinsight-spark.sif: with-spark.def mobileinsight.sif + "$(APPTAINER)" build "$@" "$<" + prepare: @mkdir -p "$(OVERLAY_DIR)" @lsmod | grep overlay > /dev/null || sudo modprobe overlay @@ -17,6 +20,10 @@ run: mobileinsight.sif prepare @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ -B "/run/user/$(UID)" mobileinsight.sif +run-spark: mobileinsight-spark.sif prepare + @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ + -B "/run/user/$(UID)" mobileinsight-spark.sif + clean: rm -rf *.sif $(OVERLAY_DIR) diff --git a/with-spark.def b/with-spark.def new file mode 100644 index 0000000..101bed4 --- /dev/null +++ b/with-spark.def @@ -0,0 +1,32 @@ +Bootstrap: localimage +From: mobileinsight.sif + +%arguments + SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" + +%environment + export SPARK_HOME="/opt/spark" + export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH" + export PYSPARK_PYTHON="/usr/bin/python3" + export SPARK_LOCAL_IP="127.0.0.1" + +%post + # Update and install dependencies + mkdir -p /build + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get upgrade -y + apt-get install -y \ + openjdk-8-jre-headless \ + scala + + # Download Spark + SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")" + cd /build + wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ" + tar -xf "$SPARK_TGZ" + mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark + + # Cleanup + apt-get autoclean -y + rm -rf /build