Venv2

.gitignore

@@ -1,2 +1,3 @@
 # ANSIBLE_HOME, if it's set here.
 .ansible/
+.virtualenv/

README.md

@@ -22,8 +22,8 @@ Tested on Sequoia. Requires python3.
 
 1. Run `init.sh`, which does the following:
     1. Accepts the Xcode license (may prompt for admin password)
-    1. Installs ansible (`pip3 install --user ansible`)
-    1. Installs task dependencies in `requirements.txt` (again, with `pip3 install --user`).
+    1. Installs ansible in a virtual environment (in the `.virtualenv` subdirectory)
+    1. Installs task dependencies in `requirements.txt` (in the virtual environment)
     1. Installs dependencies in `requirements.yml` using `ansible-galaxy`.
 1. Optionally grant users the ability to use sudo with `sudoers.sh -K`. See `sudoers-playbook.yml` for
    options.
@@ -79,6 +79,20 @@ Tested on Sequoia. Requires python3.
 1. [JetBrains Mono](https://www.jetbrains.com/lp/mono/)
 1. [DejaVu](https://dejavu-fonts.github.io/)
 
+## Virtual environment
+
+Our setup uses a python [virtual environment](https://docs.python.org/3/library/venv.html), created when you run
+`init.sh`. The python executable, libraries, etc., are _copied_ into this environment, protecting us against changes to
+the original files (e.g., if we upgrade the macports python version, we risk breaking symlinks in a linked virtual
+environment, whereas copies will continue to function).
+
+Each run of `init.sh` will try to create this virtual environment if it does not already exist. This is determined by
+checking for an executable `pip3` in the virtual environment's `bin` directory.
+
+You can pass the `-u|--upgrade-venv` option to this script. If the version of python in use is newer than that used to
+create the virtual environment, the virtual environment will be upgraded to use the newer python version. Note that
+this fails with a (harmless) error if the virtual environment's python executable is the same as the python executable
+on the path.
 
 ## Python versions
 
@@ -96,17 +110,19 @@ steps to get properly set up is
 
 1. Run `init.sh` to install ansible for the default (`3.9`) python.
 1. Run `bootstrap.sh` to bootstrap using this installation. This installs a newer python version.
-1. Run `init.sh` again to install ansible for the new python.
+1. (Optional) Remove (or rename for backup purposes) the `3.9` virtual environment in `.virtualenv`.
+1. Run `init.sh -u` to recreate the virtual environment with the newer python version.
 1. (Optional) install the development dependencies by running `dev-init.sh`.
-1. Run `setup.sh` as needed. This will run the ansible installed for the newer python version.
+1. Run `setup.sh` as needed. This will now use the newer python virtual environment.
 
 ### Upgrading python.
 
 To upgrade python:
 
 1. First, change the version installed in the bootstrap ports. 
 1. Run `bootstrap.sh` to install this version. 
-1. Run `init.sh` to install ansible, etc., for the new python version.
+1. (Optional) Remove (or rename for backup purposes) the old virtual environment in `.virtualenv`.
+1. Run `init.sh -u` to create a new virtual environment using the new python version.
 1. After verifying that things work with the new python version, optionally remove the old version.
 
 

bootstrap.sh

@@ -4,8 +4,10 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
+source "${DIR}/common.sh"
+
 # Run bootstrap. Installs Xcode's "additional components", if necessary. Installs MacPorts and ports required by setup.
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/bootstrap-playbook.yml "$@"
+_run_ansible_playbook "${DIR}" bootstrap-playbook.yml "$@"
 
 
 # Local Variables:

common.sh

@@ -0,0 +1,47 @@
+# Common shell functions and variables.
+
+function _ansible_venv_() {
+  local script_dir="${1}"
+  shift || return 1
+  echo "${script_dir}/.virtualenv"
+}
+
+function _ansible_venv_exe_() {
+    local executable="${1}"
+    local script_dir="${2}"
+    shift || return 2
+    echo "$(_ansible_venv_ "${script_dir}")"/bin/"${executable}"
+}
+
+function _ansible_pip_() {
+    _ansible_venv_exe_ pip3 "$@"
+}
+
+function _ansible_galaxy_() {
+    _ansible_venv_exe_ ansible-galaxy "$@"
+}
+
+function _ansible_playbook_() {
+    _ansible_venv_exe_ ansible-playbook "$@"
+}
+
+function _ansible_home_() {
+    local script_dir="${1}"
+    shift || return 1
+
+    echo "${script_dir}/.ansible"
+}
+
+function _run_ansible_playbook_() {
+    local script_dir="${1}"
+    local playbook="${2}"
+    shift 2 || return 1
+
+    local cmd
+    cmd="$(_ansible_playbook_ "${script_dir}")"
+
+    ANSIBLE_HOME="$(_ansible_home_ "${script_dir}")"
+    export ANSIBLE_HOME
+
+    "${cmd}" "${script_dir}/${playbook}" "$@"
+}

dev-init.sh

@@ -6,7 +6,11 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
-pip3 install --user --upgrade -r "${DIR}"/dev-requirements.txt --no-warn-script-location
+source "${DIR}/common.sh"
+
+PIP="$(_ansible_pip_ "${DIR}")"
+
+"${PIP}" install --upgrade -r "${DIR}"/dev-requirements.txt --no-warn-script-location
 
 
 # Local Variables:

init.sh

@@ -4,6 +4,9 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
+zparseopts -D -E -F -- \
+  u=UPGRADE -upgrade-venv=UPGRADE
+
 # Accept the Xcode license, install python and ansible-galaxy dependencies, including ansible itself.
 # Can be run again to upgrade any of the python or ansible-galaxy dependencies.
 
@@ -16,15 +19,37 @@ if ! xcodebuild -checkFirstLaunchStatus; then
   echo "Done."
 fi
 
+source "${DIR}/common.sh"
+
+VENV="$(_ansible_venv_ "${DIR}")"
+PIP="$(_ansible_pip_ "${DIR}")"
+GALAXY="$(_ansible_galaxy_ "${DIR}")"
+
+ANSIBLE_HOME="$(_ansible_home "${DIR}")"
+export ANSIBLE_HOME
+
+if [[ ! -e "${PIP}" ]]; then
+  # Create the virtual environment. This will use whatever version of python is on the path, and will copy
+  # that python and its libraries into the virtualenv rather than linking them.
+  python3 -m venv --copies --without-scm-ignore-files "${VENV}"
+elif (( $#UPGRADE > 0 )); then
+    echo "Upgrading virtual environment..."
+    # TODO Annoyingly, this prints an error if the python version is the same in the path and virtualenv...
+    python3 -m venv --copies --upgrade --without-scm-ignore-files "${VENV}"
+    echo "Done."
+else
+  echo "Skipping virtual environment creation/upgrade."
+fi
+
 # Install the latest version of pip. Some older versions won't download cryptography wheels for some reason, causing
 # the ansible install to fail.
-pip3 install --user --upgrade pip --no-warn-script-location
+"${PIP}" install --upgrade pip --no-warn-script-location
 
 # Install ansible and python requirements needed by tasks used in these playbooks.
-pip3 install --user --upgrade -r "${DIR}"/requirements.txt --no-warn-script-location
+"${PIP}" install --upgrade -r "${DIR}"/requirements.txt --no-warn-script-location
 
 # Install playbook requirements from ansible galaxy.
-"$(python3 -m site --user-base)"/bin/ansible-galaxy install -r "${DIR}"/requirements.yml
+"${GALAXY}" install -r "${DIR}"/requirements.yml
 
 
 # Local Variables:

requirements.txt

@@ -1,4 +1,4 @@
 # Python requirements for ansible tasks used in this playbook. Will be installed by init.sh
 
-ansible  # Obviously.
+ansible # Obviously.
 macholib # Dependency of `emacs` role. Used to make emacs standalone.

setup.sh

@@ -11,8 +11,10 @@ DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 # defined here.
 export PATH=/opt/local/bin:/opt/local/sbin:"${PATH}"
 
-# Assumes ansible was installed with pip3 --user (see init.sh).
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/setup-playbook.yml "$@"
+source "${DIR}/common.sh"
+
+# Run the setup playbook with any provided arguments.
+_run_ansible_playbook_ "${DIR}" setup-playbook.yml "$@"
 
 
 # Local Variables:

sudoers.sh

@@ -7,8 +7,10 @@
 SCRIPT="${(%):-%x}"
 DIR="$( cd "$( dirname "${SCRIPT}" )" >/dev/null 2>&1 && pwd )"
 
-# Assumes ansible was installed with pip3 --user (see init.sh).
-"$(python3 -m site --user-base)"/bin/ansible-playbook "${DIR}"/sudoers-playbook.yml "$@"
+source "${DIR}/common.sh"
+
+# Run the sudoers playbook with any provided arguments.
+_run_ansible_playbook_ "${DIR}" sudoers-playbook.yml "$@"
 
 
 # Local Variables: