Try using uv for python packages

vm-ubuntu-24.04/install-p4dev-v8.sh

@@ -140,6 +140,32 @@ max_of_list() {
     echo ${max}
 }
 
+dump_python_lib_info() {
+    local output_dir=$1
+    set +e
+    mkdir -p ${output_dir}
+    outf=${output_dir}/dirs.txt
+    echo "All directories named site-packages or dist-packages:" > ${outf}
+    find / -name site-packages -o -name dist-packages | sort >> ${outf}
+    echo "" >> ${outf}
+    echo "ls -la on each such directory:" >> ${outf}
+    for d in $(find / -name site-packages -o -name dist-packages | sort)
+    do
+	echo $d >> ${outf}
+	ls -la $d >> ${outf}
+    done
+    #echo "" >> ${outf}
+    #echo "ls -laR on each such directory:" >> ${outf}
+    for d in $(find / -name site-packages -o -name dist-packages | sort)
+    do
+	outf="${output_dir}/ls-laR-of-$(echo "$d" | tr '/' '-')"
+	cp /dev/null ${outf}
+	#echo $d >> ${outf}
+	ls -laR $d >> ${outf}
+    done
+    set -e
+}
+
 # Change this to a lower value if you do not like all this extra debug
 # output.  It is occasionally useful to debug why Python package
 # install files, or other files installed system-wide, are not going
@@ -474,6 +500,10 @@ pip -V  || echo "No such command in PATH: pip"
 pip2 -V || echo "No such command in PATH: pip2"
 pip3 -V || echo "No such command in PATH: pip3"
 
+DEBUG_DUMP_DIR="${HOME}/install-p4dev-dumpdir"
+mkdir -p ${DEBUG_DUMP_DIR}
+dump_python_lib_info "${DEBUG_DUMP_DIR}/000-init"
+
 # On new systems if you have never checked repos you should do that first
 
 # Install a few packages (vim is not strictly necessary -- installed for
@@ -498,6 +528,7 @@ fi
 TIME_AUTOTOOLS_START=$(date +%s)
 if [ "${ID}" = "ubuntu" ]
 then
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/003-just-before-python3-pip-install"
     sudo apt-get --yes install \
 	 autoconf automake libtool curl make g++ unzip \
 	 pkg-config python3-pip python3-venv
@@ -508,6 +539,8 @@ then
 	 pkg-config python3-pip
 fi
 
+dump_python_lib_info "${DEBUG_DUMP_DIR}/005-after-python-apt-installs"
+
 if [ \( "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" \) -o \( "${ID}" = "fedora" -a "${VERSION_ID}" = "35" \) ]
 then
     if [ -d automake-1.16.5 ]
@@ -561,11 +594,21 @@ TIME_AUTOTOOLS_END=$(date +%s)
 echo "autotools              : $(($TIME_AUTOTOOLS_END-$TIME_AUTOTOOLS_START)) sec"
 DISK_USED_AFTER_AUTOTOOLS=`get_used_disk_space_in_mbytes`
 
+# Install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+echo $PATH
+source $HOME/.local/bin/env
+echo $PATH
+which uv
+uv pip list
+
 # Create a new Python virtual environment using venv.  Later we will
 # attempt to ensure that all new Python packages installed are
 # installed into this virtual environment, not into system-wide
 # directories like /usr/local/bin
-python3 -m venv "${PYTHON_VENV}"
+dump_python_lib_info "${DEBUG_DUMP_DIR}/008-just-before-venv-creation"
+#python3 -m venv "${PYTHON_VENV}"
+uv venv "${PYTHON_VENV}"
 source "${PYTHON_VENV}/bin/activate"
 
 pip -V  || echo "No such command in PATH: pip"
@@ -579,8 +622,11 @@ pip3 -V || echo "No such command in PATH: pip3"
 pip list  || echo "Some error occurred attempting to run command: pip"
 pip3 list || echo "Some error occurred attempting to run command: pip3"
 
+uv pip list
+
 cd "${INSTALL_DIR}"
 debug_dump_many_install_files ${INSTALL_DIR}/usr-local-1-before-protobuf.txt
+dump_python_lib_info "${DEBUG_DUMP_DIR}/010-before-installing-grpc"
 
 set +x
 echo "------------------------------------------------------------"
@@ -597,18 +643,22 @@ then
     sudo apt-get --yes install libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc-dev libgrpc++-dev
     if [ "${PROTOBUF_VERSION_FOR_PIP}" != "" ]
     then
-	pip3 install protobuf==${PROTOBUF_VERSION_FOR_PIP}
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-installing-protobuf-via-pip"
+	uv pip install protobuf==${PROTOBUF_VERSION_FOR_PIP}
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/015-after-installing-protobuf-via-pip"
     fi
     TIME_GRPC_INSTALL_END=$(date +%s)
-    pip3 list
+    uv pip list
 else
     # Do not bother installing protobuf package from source code, as
     # whatever parts of protobuf we need is installed as a result of
     # installing grpc from source code, and/or installing the Python
     # protobuf package using pip.
     if [ "${PROTOBUF_VERSION_FOR_PIP}" != "" ]
     then
-	pip3 install protobuf==${PROTOBUF_VERSION_FOR_PIP}
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-installing-protobuf-via-pip"
+	uv pip install protobuf==${PROTOBUF_VERSION_FOR_PIP}
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/015-after-installing-protobuf-via-pip"
     fi
 
     cd "${INSTALL_DIR}"
@@ -673,9 +723,11 @@ else
 	# https://github.com/grpc/grpc/issues/30524
 	cmake ../.. -DgRPC_SSL_PROVIDER=package
 	make
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-grpc-sudo-make-install"
 	sudo make install
 	cd ../..
 	sudo ldconfig
+	dump_python_lib_info "${DEBUG_DUMP_DIR}/020-after-installing-grpc-from-source"
 	# Without the following command, later the command 'pkg-config
 	# --cflags grpc' fails, at least on Ubuntu 23.10 after building
 	# grpc v1.54.2
@@ -733,6 +785,8 @@ echo "start install PI:"
 set -x
 date
 
+dump_python_lib_info "${DEBUG_DUMP_DIR}/025-before-PI"
+
 TIME_PI_CLONE_START=$(date +%s)
 TIME_PI_CLONE_END=$(date +%s)
 TIME_PI_INSTALL_START=$(date +%s)
@@ -783,7 +837,9 @@ else
     /usr/local/bin/protoc --version
     set -e
     make
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/027-PI-just-before-sudo-make-install"
     sudo make install
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/029-PI-just-after-sudo-make-install"
 
     DISK_USED_BEFORE_PI_CLEANUP=`get_used_disk_space_in_mbytes`
     if [ ${CLEAN_UP_AS_WE_GO} -eq 1 ]
@@ -807,6 +863,8 @@ echo "end install PI:"
 set -x
 date
 
+dump_python_lib_info "${DEBUG_DUMP_DIR}/030-before-behavioral-model"
+
 cd "${INSTALL_DIR}"
 debug_dump_many_install_files ${INSTALL_DIR}/usr-local-4-after-PI.txt
 
@@ -853,7 +911,9 @@ else
     patch -p1 < "${PATCH_DIR}/behavioral-model-support-venv-2026-apr.patch"
     # This command installs Thrift, which I want to include in my build of
     # simple_switch_grpc
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/031-behavioral-model-just-before-install_deps"
     ./install_deps.sh
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/035-behavioral-model-just-after-install_deps"
     # simple_switch_grpc README.md says to configure and build the bmv2
     # code first, using these commands:
     ./autogen.sh
@@ -866,11 +926,14 @@ else
 	PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure --with-pi --with-thrift ${configure_python_prefix} 'CXXFLAGS=-O0 -g'
     fi
     make
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/038-behavioral-model-just-before-sudo-make-install-strip"
     sudo make install-strip
     sudo ldconfig
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/040-after-behavioral-model-install"
     # 'sudo make install-strip' installs several files in ${PYTHON_VENV}
     # with root owner.  Change them to be owned by the regular user id.
     change_owner_and_group_of_venv_lib_python3_files ${PYTHON_VENV}
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/045-after-behavioral-model-change_owner"
     DISK_USED_BEFORE_BMV2_CLEANUP=`get_used_disk_space_in_mbytes`
     if [ ${CLEAN_UP_AS_WE_GO} -eq 1 ]
     then
@@ -894,6 +957,8 @@ date
 cd "${INSTALL_DIR}"
 debug_dump_many_install_files ${INSTALL_DIR}/usr-local-5-after-behavioral-model.txt
 
+dump_python_lib_info "${DEBUG_DUMP_DIR}/050-before-p4c"
+
 set +x
 echo "------------------------------------------------------------"
 echo "Installing p4lang/p4c"
@@ -928,8 +993,10 @@ fi
 # TODO: It appears that some changes were made from scapy 2.5.0 to
 # 2.6.0 that require changes in P4 open source tools in order to use
 # version 2.6.0.  Until those changes are made, install scapy 2.5.0.
-pip3 install scapy==2.5.0 ply
-pip3 list
+uv pip install scapy==2.5.0 ply
+uv pip list
+
+dump_python_lib_info "${DEBUG_DUMP_DIR}/055-before-p4c-after-install-scapy"
 
 DISK_USED_BEFORE_P4C_CLEANUP=`get_used_disk_space_in_mbytes`
 if [ -d p4c ]
@@ -956,7 +1023,9 @@ else
     cmake .. -DCMAKE_BUILD_TYPE=Release ${P4C_CMAKE_OPTS}
     MAX_PARALLEL_JOBS=`max_parallel_jobs 2048`
     make -j${MAX_PARALLEL_JOBS}
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/058-p4c-just-before-sudo-make-install-strip"
     sudo make install/strip
+    dump_python_lib_info "${DEBUG_DUMP_DIR}/060-p4c-just-after-sudo-make-install-strip"
     sudo ldconfig
     DISK_USED_BEFORE_P4C_CLEANUP=`get_used_disk_space_in_mbytes`
     if [ ${CLEAN_UP_AS_WE_GO} -eq 1 -a ${KEEP_P4C_BUILD_FOR_TESTING} -eq 0 ]
@@ -1014,7 +1083,9 @@ then
     sudo mv /etc/sudoers.d/sudoers-dotfiles /etc/sudoers.d/sudoers-dotfiles.orig
     RESTORE_SUDOERS_FILE=1
 fi
+dump_python_lib_info "${DEBUG_DUMP_DIR}/065-mininet-just-before-install"
 PYTHON=python3 ./mininet/util/install.sh -nw
+dump_python_lib_info "${DEBUG_DUMP_DIR}/070-mininet-just-after-install"
 if [ ${RESTORE_SUDOERS_FILE} -eq 1 ]
 then
     sudo mv /etc/sudoers.d/sudoers-dotfiles.orig /etc/sudoers.d/sudoers-dotfiles
@@ -1046,13 +1117,16 @@ TIME_PTF_START=$(date +%s)
 # Ubuntu 22.04.
 #sudo pip3 install pypcap
 
+dump_python_lib_info "${DEBUG_DUMP_DIR}/075-before-ptf"
 git clone https://github.com/p4lang/ptf
 cd ptf
 if [ "x${INSTALL_PTF_SOURCE_VERSION}" != "x" ]; then
     git checkout ${INSTALL_PTF_SOURCE_VERSION}
 fi
 git log -n 1
-pip install .
+dump_python_lib_info "${DEBUG_DUMP_DIR}/078-ptf-just-before-pip-install"
+uv pip install .
+dump_python_lib_info "${DEBUG_DUMP_DIR}/080-ptf-just-after-pip-install"
 TIME_PTF_END=$(date +%s)
 echo "p4lang/ptf             : $(($TIME_PTF_END-$TIME_PTF_START)) sec"
 
@@ -1079,7 +1153,9 @@ elif [ "${ID}" = "fedora" ]
 then
     sudo dnf -y install gflags-devel net-tools
 fi
-pip3 install psutil crcmod
+dump_python_lib_info "${DEBUG_DUMP_DIR}/084-before-pip-install-psutil-crcmod"
+uv pip install psutil crcmod
+dump_python_lib_info "${DEBUG_DUMP_DIR}/085-after-pip-install-psutil-crcmod"
 
 # Install p4runtime-shell from source repo, with a slightly modified
 # setup.cfg file so that it allows us to keep the version of the
@@ -1092,15 +1168,17 @@ pip3 install psutil crcmod
 # First install a known working version of the grpcio package, because
 # otherwise installing p4runtime-shell packages will likely pick some
 # very recent version of grpcio that may cause trouble.
-pip3 install wheel
+uv pip install wheel
+dump_python_lib_info "${DEBUG_DUMP_DIR}/090-after-pip-install-wheel"
 if [ "${ID}" == "ubuntu" -a "${VERSION_ID}" == "24.04" ]
 then
     # Version 1.51.3 fails to install on Ubuntu 24.04 as of
     # 2024-May-20.
-    pip3 install grpcio==1.59.3
+    uv pip install grpcio==1.59.3
 else
-    pip3 install grpcio==1.51.3
+    uv pip install grpcio==1.51.3
 fi
+dump_python_lib_info "${DEBUG_DUMP_DIR}/095-after-pip-install-grpcio"
 
 git clone https://github.com/p4lang/p4runtime-shell
 cd p4runtime-shell
@@ -1110,7 +1188,9 @@ fi
 git log -n 1
 PATCH_DIR="${THIS_SCRIPT_DIR_ABSOLUTE}/patches"
 patch -p1 < "${PATCH_DIR}/p4runtime-shell-2023-changes.patch"
-pip3 install .
+dump_python_lib_info "${DEBUG_DUMP_DIR}/099-p4runtime-shell-just-before-pip-install"
+uv pip install .
+dump_python_lib_info "${DEBUG_DUMP_DIR}/100-p4runtime-shell-just-after-pip-install"
 
 pip3 list
 

vm-ubuntu-24.04/patches/behavioral-model-adjust-ubuntu-packges.patch

@@ -1,5 +1,5 @@
 diff --git a/install_deps.sh b/install_deps.sh
-index 02dd6d2..7673969 100755
+index 02dd6d2..f1510fe 100755
 --- a/install_deps.sh
 +++ b/install_deps.sh
 @@ -18,31 +18,20 @@ sudo apt-get install -y \
@@ -35,3 +35,11 @@ index 02dd6d2..7673969 100755
      wget
 
  tmpdir=`mktemp -d -p .`
+@@ -50,7 +39,6 @@ cd $tmpdir
+
+ bash ../ci/install-thrift.sh
+ bash ../ci/install-nanomsg.sh
+-bash ../ci/install-pi.sh
+ sudo ldconfig
+ bash ../ci/install-pynng.sh
+