From 8943eceb93f7063b0af0d72924779699522f36d6 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 2 May 2024 12:29:44 -0700 Subject: [PATCH 1/8] Removed spark implementation, use a docker-compose cluster for testing spark --- Makefile | 7 ------- with-spark.def | 32 -------------------------------- 2 files changed, 39 deletions(-) delete mode 100644 with-spark.def diff --git a/Makefile b/Makefile index b941ad1..7c658f3 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,6 @@ UID := $(shell id -u) mobileinsight.sif: mobileinsight.def "$(APPTAINER)" build --build-arg="NUM_THREADS=$(NUM_THREADS)" "$@" "$<" -mobileinsight-spark.sif: with-spark.def mobileinsight.sif - "$(APPTAINER)" build "$@" "$<" - prepare: @mkdir -p "$(OVERLAY_DIR)" @lsmod | grep overlay > /dev/null || sudo modprobe overlay @@ -20,10 +17,6 @@ run: mobileinsight.sif prepare @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ -B "/run/user/$(UID)" mobileinsight.sif -run-spark: mobileinsight-spark.sif prepare - @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ - -B "/run/user/$(UID)" mobileinsight-spark.sif - clean: rm -rf *.sif $(OVERLAY_DIR) diff --git a/with-spark.def b/with-spark.def deleted file mode 100644 index 101bed4..0000000 --- a/with-spark.def +++ /dev/null @@ -1,32 +0,0 @@ -Bootstrap: localimage -From: mobileinsight.sif - -%arguments - SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" - -%environment - export SPARK_HOME="/opt/spark" - export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH" - export PYSPARK_PYTHON="/usr/bin/python3" - export SPARK_LOCAL_IP="127.0.0.1" - -%post - # Update and install dependencies - mkdir -p /build - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get upgrade -y - apt-get install -y \ - openjdk-8-jre-headless \ - scala - - # Download Spark - SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")" - cd /build - wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ" - tar -xf "$SPARK_TGZ" - mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark - - # Cleanup - apt-get autoclean -y - rm -rf /build From 73002367914e59207110443ae14b4841a6da05db Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 2 May 2024 19:04:38 -0700 Subject: [PATCH 2/8] Revert "Removed spark implementation, use a docker-compose cluster for testing" This reverts commit 8943eceb93f7063b0af0d72924779699522f36d6. --- Makefile | 7 +++++++ with-spark.def | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 with-spark.def diff --git a/Makefile b/Makefile index 7c658f3..b941ad1 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ UID := $(shell id -u) mobileinsight.sif: mobileinsight.def "$(APPTAINER)" build --build-arg="NUM_THREADS=$(NUM_THREADS)" "$@" "$<" +mobileinsight-spark.sif: with-spark.def mobileinsight.sif + "$(APPTAINER)" build "$@" "$<" + prepare: @mkdir -p "$(OVERLAY_DIR)" @lsmod | grep overlay > /dev/null || sudo modprobe overlay @@ -17,6 +20,10 @@ run: mobileinsight.sif prepare @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ -B "/run/user/$(UID)" mobileinsight.sif +run-spark: mobileinsight-spark.sif prepare + @$(APPTAINER) run --overlay "$(OVERLAY_DIR)" \ + -B "/run/user/$(UID)" mobileinsight-spark.sif + clean: rm -rf *.sif $(OVERLAY_DIR) diff --git a/with-spark.def b/with-spark.def new file mode 100644 index 0000000..101bed4 --- /dev/null +++ b/with-spark.def @@ -0,0 +1,32 @@ +Bootstrap: localimage +From: mobileinsight.sif + +%arguments + SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" + +%environment + export SPARK_HOME="/opt/spark" + export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH" + export PYSPARK_PYTHON="/usr/bin/python3" + export SPARK_LOCAL_IP="127.0.0.1" + +%post + # Update and install dependencies + mkdir -p /build + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get upgrade -y + apt-get install -y \ + openjdk-8-jre-headless \ + scala + + # Download Spark + SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")" + cd /build + wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ" + tar -xf "$SPARK_TGZ" + mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark + + # Cleanup + apt-get autoclean -y + rm -rf /build From 37c4bad792f9ed8c09e8b5d93417ae6dc27f76c0 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 2 May 2024 19:27:04 -0700 Subject: [PATCH 3/8] Simplified spark version so pyspark can work --- with-spark.def | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/with-spark.def b/with-spark.def index 101bed4..21aea93 100644 --- a/with-spark.def +++ b/with-spark.def @@ -5,9 +5,6 @@ From: mobileinsight.sif SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" %environment - export SPARK_HOME="/opt/spark" - export PATH="/opt/spark/sbin:/opt/spark/bin:$PATH" - export PYSPARK_PYTHON="/usr/bin/python3" export SPARK_LOCAL_IP="127.0.0.1" %post @@ -19,14 +16,7 @@ From: mobileinsight.sif apt-get install -y \ openjdk-8-jre-headless \ scala - - # Download Spark - SPARK_TGZ="/build/$(basename "{{ SPARK_URL }}")" - cd /build - wget "{{ SPARK_URL }}" -nv -O "$SPARK_TGZ" - tar -xf "$SPARK_TGZ" - mv "$(echo $SPARK_TGZ | sed 's/\.tgz$//')" /opt/spark + pip3 install pyspark==3.5.1 # Cleanup apt-get autoclean -y - rm -rf /build From 8f1a608c66b2a0bddd484c6c794c74436b9f5818 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 2 May 2024 19:43:20 -0700 Subject: [PATCH 4/8] Remove unused /build dir --- with-spark.def | 7 ------- 1 file changed, 7 deletions(-) diff --git a/with-spark.def b/with-spark.def index 21aea93..69f0afc 100644 --- a/with-spark.def +++ b/with-spark.def @@ -1,15 +1,8 @@ Bootstrap: localimage From: mobileinsight.sif -%arguments - SPARK_URL="https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz" - -%environment - export SPARK_LOCAL_IP="127.0.0.1" - %post # Update and install dependencies - mkdir -p /build export DEBIAN_FRONTEND=noninteractive apt-get update apt-get upgrade -y From c1b65d744e339f74566f6d8b882cf43db87ca9b1 Mon Sep 17 00:00:00 2001 From: Yiyao Yu Date: Fri, 17 May 2024 22:27:18 -0700 Subject: [PATCH 5/8] Use upstream wireshark to overcome MobileInsight website availability --- mobileinsight.def | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobileinsight.def b/mobileinsight.def index 84b3263..a1eaa28 100644 --- a/mobileinsight.def +++ b/mobileinsight.def @@ -45,8 +45,8 @@ From: ubuntu:22.04 # Download resources cd /build - wget "http://www.mobileinsight.net/wireshark-{{ WS_VER }}-rbc-dissector.tar.xz" -nv \ - -O "wireshark-{{ WS_VER }}.tar.xz" + wget "https://github.com/wireshark/wireshark/archive/refs/tags/v{{ WS_VER }}.tar.gz" \ + -nv -O "wireshark-{{ WS_VER }}.tar.xz" tar -xf "wireshark-{{ WS_VER }}.tar.xz" git clone "https://github.com/mobile-insight/mobileinsight-core.git" git -C mobileinsight-core checkout "{{ MI_COMMIT }}" From 86a52a6f52a198285a28b33925ae75caf1abaa14 Mon Sep 17 00:00:00 2001 From: Yiyao Yu Date: Fri, 17 May 2024 22:27:52 -0700 Subject: [PATCH 6/8] Add pandas to Spark dependency --- with-spark.def | 1 + 1 file changed, 1 insertion(+) diff --git a/with-spark.def b/with-spark.def index 69f0afc..512fbcd 100644 --- a/with-spark.def +++ b/with-spark.def @@ -7,6 +7,7 @@ From: mobileinsight.sif apt-get update apt-get upgrade -y apt-get install -y \ + python3-pandas \ openjdk-8-jre-headless \ scala pip3 install pyspark==3.5.1 From 24096b896dc52561d109845d352437409129de2d Mon Sep 17 00:00:00 2001 From: Yiyao Yu Date: Sat, 18 May 2024 18:38:25 -0700 Subject: [PATCH 7/8] Added python dill to spark variant --- with-spark.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/with-spark.def b/with-spark.def index 512fbcd..16cdf96 100644 --- a/with-spark.def +++ b/with-spark.def @@ -10,7 +10,7 @@ From: mobileinsight.sif python3-pandas \ openjdk-8-jre-headless \ scala - pip3 install pyspark==3.5.1 + pip3 install pyspark[sql]==3.5.1 dill==0.3.8 # Cleanup apt-get autoclean -y From 4e3942eb31db5a02f967de0b72408dc07bbb8075 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 21 May 2024 15:14:56 -0700 Subject: [PATCH 8/8] Apply patch to fix aww.proto registration in WS dissector --- 00-cpp_compat.patch | 4 ++-- 01-radio_bearer_config.patch | 25 +++++++++++++++++++++++++ mobileinsight.def | 5 ++++- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 01-radio_bearer_config.patch diff --git a/00-cpp_compat.patch b/00-cpp_compat.patch index 52818c4..1060f11 100644 --- a/00-cpp_compat.patch +++ b/00-cpp_compat.patch @@ -1,5 +1,5 @@ ---- epan.h.old 2024-04-20 00:34:15.534364916 -0700 -+++ epan.h 2024-04-20 00:34:25.094467480 -0700 +--- a/epan/epan.h 2024-04-20 00:34:15.534364916 -0700 ++++ b/epan/epan.h 2024-04-20 00:34:25.094467480 -0700 @@ -10,9 +10,6 @@ #ifndef __EPAN_H__ #define __EPAN_H__ diff --git a/01-radio_bearer_config.patch b/01-radio_bearer_config.patch new file mode 100644 index 0000000..b3444a4 --- /dev/null +++ b/01-radio_bearer_config.patch @@ -0,0 +1,25 @@ +diff --git a/epan/dissectors/asn1/nr-rrc/nr-rrc.cnf b/epan/dissectors/asn1/nr-rrc/nr-rrc.cnf +index fd478d8969..b3b89250e0 100644 +--- a/epan/dissectors/asn1/nr-rrc/nr-rrc.cnf ++++ b/epan/dissectors/asn1/nr-rrc/nr-rrc.cnf +@@ -145,7 +145,7 @@ PH-TypeListMCG + PH-TypeListSCG + RA-ReportList-r16 + RACH-ConfigCommon +-RadioBearerConfig ++RadioBearerConfig @radio_bearer_conf + ReferenceTime-r16 + RRCReconfiguration @rrc_reconf + RRCReconfigurationComplete +diff --git a/epan/dissectors/packet-nr-rrc.c b/epan/dissectors/packet-nr-rrc.c +index 75977474c0..cfa8602711 100644 +--- a/epan/dissectors/packet-nr-rrc.c ++++ b/epan/dissectors/packet-nr-rrc.c +@@ -112214,6 +112214,7 @@ proto_register_nr_rrc(void) { + register_dissector("nr-rrc.ue_nr_cap", dissect_nr_rrc_UE_NR_Capability_PDU, proto_nr_rrc); + register_dissector("nr-rrc.sbcch.sl.bch", dissect_SBCCH_SL_BCH_Message_PDU, proto_nr_rrc); + register_dissector("nr-rrc.scch", dissect_SCCH_Message_PDU, proto_nr_rrc); ++ register_dissector("nr-rrc.radio_bearer_conf", dissect_nr_rrc_RadioBearerConfig_PDU, proto_nr_rrc); + + + /*--- End of included file: packet-nr-rrc-dis-reg.c ---*/ diff --git a/mobileinsight.def b/mobileinsight.def index a1eaa28..810c4ab 100644 --- a/mobileinsight.def +++ b/mobileinsight.def @@ -8,6 +8,7 @@ From: ubuntu:22.04 %files 00-cpp_compat.patch /build/00-cpp_compat.patch + 01-radio_bearer_config.patch /build/01-radio_bearer_config.patch %post # Update and install dependencies @@ -53,7 +54,9 @@ From: ubuntu:22.04 # Compile and install wireshark cd "/build/wireshark-{{ WS_VER }}" - patch epan/epan.h < /build/00-cpp_compat.patch + for patch in /build/*.patch; do + patch -p1 < "$patch" + done cmake -DBUILD_wireshark=OFF . make -j "{{ NUM_THREADS }}" make install