From 246b7c8282420e459ce941ce101c10facebfe557 Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Mon, 22 Sep 2025 17:55:09 -0400
Subject: [PATCH 1/6] Start switch to venv.

---
 .gitignore          |  1 +
 .idea/mac-setup.iml |  1 +
 README.md           | 12 +++++++-----
 bootstrap.sh        |  5 ++++-
 common.sh           | 25 +++++++++++++++++++++++++
 dev-init.sh         |  6 +++++-
 init.sh             | 15 ++++++++++++---
 requirements.txt    |  2 +-
 setup.sh            |  7 +++++--
 sudoers.sh          |  7 +++++--
 10 files changed, 66 insertions(+), 15 deletions(-)
 create mode 100644 common.sh

diff --git a/.gitignore b/.gitignore
index 1fd9f3e..b59f103 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 # ANSIBLE_HOME, if it's set here.
 .ansible/
+.virtualenv/
diff --git a/.idea/mac-setup.iml b/.idea/mac-setup.iml
index 0e0325b..9a687e3 100644
--- a/.idea/mac-setup.iml
+++ b/.idea/mac-setup.iml
@@ -4,6 +4,7 @@
     <exclude-output />
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.ansible" />
+      <excludeFolder url="file://$MODULE_DIR$/.virtualenv" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
diff --git a/README.md b/README.md
index bf2dc22..716991e 100644
--- a/README.md
+++ b/README.md
@@ -22,8 +22,8 @@ Tested on Sequoia. Requires python3.
 
 1. Run `init.sh`, which does the following:
     1. Accepts the Xcode license (may prompt for admin password)
-    1. Installs ansible (`pip3 install --user ansible`)
-    1. Installs task dependencies in `requirements.txt` (again, with `pip3 install --user`).
+    1. Installs ansible in a virtual environment (in the `.virtualenv` subdirectory)
+    1. Installs task dependencies in `requirements.txt` (in the virtual environment)
     1. Installs dependencies in `requirements.yml` using `ansible-galaxy`.
 1. Optionally grant users the ability to use sudo with `sudoers.sh -K`. See `sudoers-playbook.yml` for
    options.
@@ -96,9 +96,10 @@ steps to get properly set up is
 
 1. Run `init.sh` to install ansible for the default (`3.9`) python.
 1. Run `bootstrap.sh` to bootstrap using this installation. This installs a newer python version.
-1. Run `init.sh` again to install ansible for the new python.
+1. Remove (or rename) the `3.9` virtual environment in `.virtualenv`.
+1. Run `init.sh` again to recreate the virtual environment with the newer python version.
 1. (Optional) install the development dependencies by running `dev-init.sh`.
-1. Run `setup.sh` as needed. This will run the ansible installed for the newer python version.
+1. Run `setup.sh` as needed. This will now use the newer python virtual environment.
 
 ### Upgrading python.
 
@@ -106,7 +107,8 @@ To upgrade python:
 
 1. First, change the version installed in the bootstrap ports. 
 1. Run `bootstrap.sh` to install this version. 
-1. Run `init.sh` to install ansible, etc., for the new python version.
+1. Remove (or rename) the old virtual environment in `.virtualenv`.
+1. Run `init.sh` to create a new virtual environment using the new python version.
 1. After verifying that things work with the new python version, optionally remove the old version.
 
 
diff --git a/bootstrap.sh b/bootstrap.sh
index 0cbfe85..b3d10ef 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -4,8 +4,11 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
+source "${DIR}/common.sh"
+PLAYBOOK="$(_ansible_python_ "${DIR}")"
+
 # Run bootstrap. Installs Xcode's "additional components", if necessary. Installs MacPorts and ports required by setup.
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/bootstrap-playbook.yml "$@"
+"${PLAYBOOK}" "${DIR}"/bootstrap-playbook.yml "$@"
 
 
 # Local Variables:
diff --git a/common.sh b/common.sh
new file mode 100644
index 0000000..0177fc1
--- /dev/null
+++ b/common.sh
@@ -0,0 +1,25 @@
+# Common shell functions and variables.
+
+function _ansible_venv_() {
+  local script_dir="${1}"
+  shift || return 1
+  echo "${script_dir}/.virtualenv"
+}
+
+function _ansible_pip_() {
+    local script_dir="${1}"
+    shift || return 1
+    echo "$(_ansible_venv_ "${script_dir}")"/bin/pip3
+}
+
+function _ansible_galaxy_() {
+    local script_dir="${1}"
+    shift || return 1
+    echo "$(_ansible_venv_ "${script_dir}")"/bin/ansible-galaxy
+}
+
+function _ansible_playbook_() {
+    local script_dir="${1}"
+    shift || return 1
+    echo "$(_ansible_venv_ "${script_dir}")"/bin/ansible-playbook
+}
diff --git a/dev-init.sh b/dev-init.sh
index 909bd5b..14d304f 100755
--- a/dev-init.sh
+++ b/dev-init.sh
@@ -6,7 +6,11 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
-pip3 install --user --upgrade -r "${DIR}"/dev-requirements.txt --no-warn-script-location
+source "${DIR}/common.sh"
+
+PIP="$(_ansible_pip_ "${DIR}")"
+
+"${PIP}" install --upgrade -r "${DIR}"/dev-requirements.txt --no-warn-script-location
 
 
 # Local Variables:
diff --git a/init.sh b/init.sh
index ba697b0..8c4a1b9 100755
--- a/init.sh
+++ b/init.sh
@@ -16,15 +16,24 @@ if ! xcodebuild -checkFirstLaunchStatus; then
   echo "Done."
 fi
 
+source "${DIR}/common.sh"
+
+VENV="$(_ansible_venv_ "${DIR}")"
+PIP="$(_ansible_pip_ "${DIR}")"
+GALAXY="$(_ansible_galaxy_ "${DIR}")"
+
+# Create the virtual environment. This will use whatever version of python is on the path.
+python3 -m venv "${VENV}"
+
 # Install the latest version of pip. Some older versions won't download cryptography wheels for some reason, causing
 # the ansible install to fail.
-pip3 install --user --upgrade pip --no-warn-script-location
+"${PIP}" install --upgrade pip --no-warn-script-location
 
 # Install ansible and python requirements needed by tasks used in these playbooks.
-pip3 install --user --upgrade -r "${DIR}"/requirements.txt --no-warn-script-location
+"${PIP}" install --upgrade -r "${DIR}"/requirements.txt --no-warn-script-location
 
 # Install playbook requirements from ansible galaxy.
-"$(python3 -m site --user-base)"/bin/ansible-galaxy install -r "${DIR}"/requirements.yml
+"${GALAXY}" install -r "${DIR}"/requirements.yml
 
 
 # Local Variables:
diff --git a/requirements.txt b/requirements.txt
index 8d0a928..f2dca86 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 # Python requirements for ansible tasks used in this playbook. Will be installed by init.sh
 
-ansible  # Obviously.
+ansible # Obviously.
 macholib # Dependency of `emacs` role. Used to make emacs standalone.
diff --git a/setup.sh b/setup.sh
index d26e9f7..a4659ca 100755
--- a/setup.sh
+++ b/setup.sh
@@ -11,8 +11,11 @@ DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 # defined here.
 export PATH=/opt/local/bin:/opt/local/sbin:"${PATH}"
 
-# Assumes ansible was installed with pip3 --user (see init.sh).
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/setup-playbook.yml "$@"
+source "${DIR}/common.sh"
+PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+
+# Run the setup playbook with any provided arguments.
+"${PLAYBOOK}" "${DIR}"/setup-playbook.yml "$@"
 
 
 # Local Variables:
diff --git a/sudoers.sh b/sudoers.sh
index c6aff27..6288cd8 100755
--- a/sudoers.sh
+++ b/sudoers.sh
@@ -7,8 +7,11 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
-# Assumes ansible was installed with pip3 --user (see init.sh).
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/sudoers-playbook.yml "$@"
+source "${DIR}/common.sh"
+PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+
+# Run the sudoers playbook with any provided arguments.
+"${PLAYBOOK}" "${DIR}"/sudoers-playbook.yml "$@"
 
 
 # Local Variables:

From c01737ce76c4dcc5a9b9a0c7fe0833df026eaf39 Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Fri, 13 Jun 2025 11:59:21 -0400
Subject: [PATCH 2/6] Use copies rather than symlinks in venv and upgrade to
 current python on each run.

---
 init.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init.sh b/init.sh
index 8c4a1b9..0d2d293 100755
--- a/init.sh
+++ b/init.sh
@@ -23,7 +23,7 @@ PIP="$(_ansible_pip_ "${DIR}")"
 GALAXY="$(_ansible_galaxy_ "${DIR}")"
 
 # Create the virtual environment. This will use whatever version of python is on the path.
-python3 -m venv "${VENV}"
+python3 -m venv --copies --upgrade "${VENV}"
 
 # Install the latest version of pip. Some older versions won't download cryptography wheels for some reason, causing
 # the ansible install to fail.

From 2986d60be6728e23c2df0a347c9aed848546299d Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Fri, 13 Jun 2025 12:15:30 -0400
Subject: [PATCH 3/6] Only upgrade virtualenv when explicitly requested.

---
 README.md | 22 ++++++++++++++++++----
 init.sh   | 17 +++++++++++++++--
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 716991e..47057ac 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,20 @@ Tested on Sequoia. Requires python3.
 1. [JetBrains Mono](https://www.jetbrains.com/lp/mono/)
 1. [DejaVu](https://dejavu-fonts.github.io/)
 
+## Virtual environment
+
+Our setup uses a python [virtual environment](https://docs.python.org/3/library/venv.html), created when you run
+`init.sh`. The python executable, libraries, etc., are _copied_ into this environment, protecting us against changes to
+the original files (e.g., if we upgrade the macports python version, we risk breaking symlinks in a linked virtual
+environment, whereas copies will continue to function).
+
+Each run of `init.sh` will try to create this virtual environment if it does not already exist. This is determined by
+checking for an executable `pip3` in the virtual environment's `bin` directory.
+
+You can pass the `-u|--upgrade-venv` option to this script. If the version of python in use is newer than that used to
+create the virtual environment, the virtual environment will be upgraded to use the newer python version. Note that
+this fails with a (harmless) error if the virtual environment's python executable is the same as the python executable
+on the path.
 
 ## Python versions
 
@@ -96,8 +110,8 @@ steps to get properly set up is
 
 1. Run `init.sh` to install ansible for the default (`3.9`) python.
 1. Run `bootstrap.sh` to bootstrap using this installation. This installs a newer python version.
-1. Remove (or rename) the `3.9` virtual environment in `.virtualenv`.
-1. Run `init.sh` again to recreate the virtual environment with the newer python version.
+1. (Optional) Remove (or rename for backup purposes) the `3.9` virtual environment in `.virtualenv`.
+1. Run `init.sh -u` to recreate the virtual environment with the newer python version.
 1. (Optional) install the development dependencies by running `dev-init.sh`.
 1. Run `setup.sh` as needed. This will now use the newer python virtual environment.
 
@@ -107,8 +121,8 @@ To upgrade python:
 
 1. First, change the version installed in the bootstrap ports. 
 1. Run `bootstrap.sh` to install this version. 
-1. Remove (or rename) the old virtual environment in `.virtualenv`.
-1. Run `init.sh` to create a new virtual environment using the new python version.
+1. (Optional) Remove (or rename for backup purposes) the old virtual environment in `.virtualenv`.
+1. Run `init.sh -u` to create a new virtual environment using the new python version.
 1. After verifying that things work with the new python version, optionally remove the old version.
 
 
diff --git a/init.sh b/init.sh
index 0d2d293..41dc739 100755
--- a/init.sh
+++ b/init.sh
@@ -4,6 +4,9 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
+zparseopts -D -E -F -- \
+  u=UPGRADE -upgrade-venv=UPGRADE
+
 # Accept the Xcode license, install python and ansible-galaxy dependencies, including ansible itself.
 # Can be run again to upgrade any of the python or ansible-galaxy dependencies.
 
@@ -22,8 +25,18 @@ VENV="$(_ansible_venv_ "${DIR}")"
 PIP="$(_ansible_pip_ "${DIR}")"
 GALAXY="$(_ansible_galaxy_ "${DIR}")"
 
-# Create the virtual environment. This will use whatever version of python is on the path.
-python3 -m venv --copies --upgrade "${VENV}"
+if [[ ! -e "${PIP}" ]]; then
+  # Create the virtual environment. This will use whatever version of python is on the path, and will copy
+  # that python and its libraries into the virtualenv rather than linking them.
+  python3 -m venv --copies --without-scm-ignore-files "${VENV}"
+elif (( $#UPGRADE > 0 )); then
+    echo "Upgrading virtual environment..."
+    # TODO Annoyingly, this prints an error if the python version is the same in the path and virtualenv...
+    python3 -m venv --copies --upgrade --without-scm-ignore-files "${VENV}"
+    echo "Done."
+else
+  echo "Skipping virtual environment creation/upgrade."
+fi
 
 # Install the latest version of pip. Some older versions won't download cryptography wheels for some reason, causing
 # the ansible install to fail.

From 18a04fab4553bdaecd7f31110dede9f0bdbbc886 Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Fri, 13 Jun 2025 12:26:54 -0400
Subject: [PATCH 4/6] Set ANSIBLE_HOME.

---
 common.sh  | 7 +++++++
 init.sh    | 3 +++
 setup.sh   | 6 ++++--
 sudoers.sh | 6 ++++--
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/common.sh b/common.sh
index 0177fc1..f9157dd 100644
--- a/common.sh
+++ b/common.sh
@@ -23,3 +23,10 @@ function _ansible_playbook_() {
     shift || return 1
     echo "$(_ansible_venv_ "${script_dir}")"/bin/ansible-playbook
 }
+
+function _ansible_home_() {
+    local script_dir="${1}"
+    shift || return 1
+
+    echo "${script_dir}/.ansible"
+}
diff --git a/init.sh b/init.sh
index 41dc739..52764dc 100755
--- a/init.sh
+++ b/init.sh
@@ -25,6 +25,9 @@ VENV="$(_ansible_venv_ "${DIR}")"
 PIP="$(_ansible_pip_ "${DIR}")"
 GALAXY="$(_ansible_galaxy_ "${DIR}")"
 
+ANSIBLE_HOME="$(_ansible_home "${DIR}")"
+export ANSIBLE_HOME
+
 if [[ ! -e "${PIP}" ]]; then
   # Create the virtual environment. This will use whatever version of python is on the path, and will copy
   # that python and its libraries into the virtualenv rather than linking them.
diff --git a/setup.sh b/setup.sh
index a4659ca..077dcb5 100755
--- a/setup.sh
+++ b/setup.sh
@@ -12,10 +12,12 @@ DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 export PATH=/opt/local/bin:/opt/local/sbin:"${PATH}"
 
 source "${DIR}/common.sh"
-PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+ANSIBLE_PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+ANSIBLE_HOME="$(_ansible_home_ "${DIR}")"
+export ANSIBLE_HOME
 
 # Run the setup playbook with any provided arguments.
-"${PLAYBOOK}" "${DIR}"/setup-playbook.yml "$@"
+"${ANSIBLE_PLAYBOOK}" "${DIR}"/setup-playbook.yml "$@"
 
 
 # Local Variables:
diff --git a/sudoers.sh b/sudoers.sh
index 6288cd8..98ef432 100755
--- a/sudoers.sh
+++ b/sudoers.sh
@@ -8,10 +8,12 @@ SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
 source "${DIR}/common.sh"
-PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+ANSIBLE_PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
+ANSIBLE_HOME="$(_ansible_home "${DIR}")"
+export ANSIBLE_HOME
 
 # Run the sudoers playbook with any provided arguments.
-"${PLAYBOOK}" "${DIR}"/sudoers-playbook.yml "$@"
+"${ANSIBLE_PLAYBOOK}" "${DIR}"/sudoers-playbook.yml "$@"
 
 
 # Local Variables:

From 8909ac59d1364bee6d25a514c7aeedd0419bdaff Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Fri, 13 Jun 2025 12:32:55 -0400
Subject: [PATCH 5/6] Simplify running playbooks by using a common function.

---
 bootstrap.sh |  3 +--
 common.sh    | 14 ++++++++++++++
 setup.sh     |  5 +----
 sudoers.sh   |  5 +----
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/bootstrap.sh b/bootstrap.sh
index b3d10ef..f08886d 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -5,10 +5,9 @@ SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
 source "${DIR}/common.sh"
-PLAYBOOK="$(_ansible_python_ "${DIR}")"
 
 # Run bootstrap. Installs Xcode's "additional components", if necessary. Installs MacPorts and ports required by setup.
-"${PLAYBOOK}" "${DIR}"/bootstrap-playbook.yml "$@"
+_run_ansible_playbook "${DIR}" bootstrap-playbook.yml "$@"
 
 
 # Local Variables:
diff --git a/common.sh b/common.sh
index f9157dd..cb8936b 100644
--- a/common.sh
+++ b/common.sh
@@ -30,3 +30,17 @@ function _ansible_home_() {
 
     echo "${script_dir}/.ansible"
 }
+
+function _run_ansible_playbook_() {
+    local script_dir="${1}"
+    local playbook="${2}"
+    shift 2 || return 1
+
+    local cmd
+    cmd="$(_ansible_playbook_ "${script_dir}")"
+
+    ANSIBLE_HOME="$(_ansible_home_ "${script_dir}")"
+    export ANSIBLE_HOME
+
+    "${cmd}" "${script_dir}/${playbook}" "$@"
+}
diff --git a/setup.sh b/setup.sh
index 077dcb5..5a156f9 100755
--- a/setup.sh
+++ b/setup.sh
@@ -12,12 +12,9 @@ DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 export PATH=/opt/local/bin:/opt/local/sbin:"${PATH}"
 
 source "${DIR}/common.sh"
-ANSIBLE_PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
-ANSIBLE_HOME="$(_ansible_home_ "${DIR}")"
-export ANSIBLE_HOME
 
 # Run the setup playbook with any provided arguments.
-"${ANSIBLE_PLAYBOOK}" "${DIR}"/setup-playbook.yml "$@"
+_run_ansible_playbook_ "${DIR}" setup-playbook.yml "$@"
 
 
 # Local Variables:
diff --git a/sudoers.sh b/sudoers.sh
index 98ef432..6399970 100755
--- a/sudoers.sh
+++ b/sudoers.sh
@@ -8,12 +8,9 @@ SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
 source "${DIR}/common.sh"
-ANSIBLE_PLAYBOOK="$(_ansible_playbook_ "${DIR}")"
-ANSIBLE_HOME="$(_ansible_home "${DIR}")"
-export ANSIBLE_HOME
 
 # Run the sudoers playbook with any provided arguments.
-"${ANSIBLE_PLAYBOOK}" "${DIR}"/sudoers-playbook.yml "$@"
+_run_ansible_playbook_ "${DIR}" sudoers-playbook.yml "$@"
 
 
 # Local Variables:

From 5fa69d109eb6e17fbd65ba9956f058d1958e7004 Mon Sep 17 00:00:00 2001
From: Mike Conley <conleym@users.noreply.github.com>
Date: Fri, 13 Jun 2025 12:47:38 -0400
Subject: [PATCH 6/6] Simplify exe path building with common function.

---
 common.sh | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/common.sh b/common.sh
index cb8936b..9839b27 100644
--- a/common.sh
+++ b/common.sh
@@ -6,22 +6,23 @@ function _ansible_venv_() {
   echo "${script_dir}/.virtualenv"
 }
 
+function _ansible_venv_exe_() {
+    local executable="${1}"
+    local script_dir="${2}"
+    shift || return 2
+    echo "$(_ansible_venv_ "${script_dir}")"/bin/"${executable}"
+}
+
 function _ansible_pip_() {
-    local script_dir="${1}"
-    shift || return 1
-    echo "$(_ansible_venv_ "${script_dir}")"/bin/pip3
+    _ansible_venv_exe_ pip3 "$@"
 }
 
 function _ansible_galaxy_() {
-    local script_dir="${1}"
-    shift || return 1
-    echo "$(_ansible_venv_ "${script_dir}")"/bin/ansible-galaxy
+    _ansible_venv_exe_ ansible-galaxy "$@"
 }
 
 function _ansible_playbook_() {
-    local script_dir="${1}"
-    shift || return 1
-    echo "$(_ansible_venv_ "${script_dir}")"/bin/ansible-playbook
+    _ansible_venv_exe_ ansible-playbook "$@"
 }
 
 function _ansible_home_() {