diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..f098155 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,37 @@ +name: ibmichroot CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 chroot_setup --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 chroot_setup --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # TODO add tests + #- name: Test with pytest + #run: | + # pytest + diff --git a/chroot_setup b/chroot_setup new file mode 100644 index 0000000..d5d3df4 --- /dev/null +++ b/chroot_setup @@ -0,0 +1,516 @@ +#!/QOpenSys/pkgs/bin/python3 + +""" +Module to automate and simplifiy chroot environment creation +""" + +import argparse +import os +import sys +import shutil +import logging +import stat +import subprocess +import re +from string import Template + +print(""" + ##### # # ###### ####### ####### ####### + # # # # # # # # # # # + # # # # # # # # # # + # ####### ###### # # # # # + # # # # # # # # # # + # # # # # # # # # # # + ##### # # # # ####### ####### # + + ##### ####### ####### # # ###### + # # # # # # # # + # # # # # # # + ##### ##### # # # ###### + # # # # # # + # # # # # # # + ##### ####### # ##### # +""") + + +def chroot_mkdir(directory: str): + """ + Makes a directory inside the chroot, as well as all intermediary + directories + """ + # :mkdir + print(f"os.makedirs({args.chroot_directory}{directory}, exist_ok=True)") + os.makedirs(f"{args.chroot_directory}{directory}", exist_ok=True) + + +def chroot_ln(line: str): + """ + Create an absolute symlink inside the chroot + """ + # We don't have to go inside the chroot, since the + # links are resolved as they are used. The target doesn't even have to + # exist, Python won't check and therefore won't throw an error. + # :ln + ln_command = line.split() + + if not len(ln_command) == 2: + raise ValueError(":ln action expects 2 arguments!") + + target, link_name = ln_command + + print(f"os.symlink({target}, {args.chroot_directory}{link_name})") + os.symlink(target, f"{args.chroot_directory}{link_name}") + + +def chroot_ln_rel(line: str): + """ + Creates a relative symlink in the chroot that mirrors a relative symlink + that exists outside of the chroot. + """ + # :ln_rel + ln_command = line.split() + + if not len(ln_command) == 1: + raise ValueError(":ln_rel action expects 1 argument!") + + directory = os.path.dirname(line) + basename = os.path.basename(line) + target = os.readlink(line) + here = os.getcwd() + print(f"os.chdir({args.chroot_directory}{directory})") + print(f"os.symlink({target}, {basename})") + os.chdir(f"{args.chroot_directory}") + os.symlink(target, basename) + os.chdir(here) + + +def chroot_ln_fix_rel(line: str): + """ + Create a relative symlink inside of the chroot, specifying where the link + should be pointed to. + """ + # This means you can change a relative symlink you pass in, or simply + # create a relative symlink where one didn't exist before + # :ln + ln_command = line.split() + + if not len(ln_command) == 2: + raise ValueError(":ln_fix_rel action expects 2 arguments!") + + target, link_name = ln_command + directory = os.path.dirname(link_name) + basename = os.path.basename(link_name) + here = os.getcwd() + print(f"os.chdir({args.chroot_directory}{directory})") + print(f"os.symlink{target}, {basename})") + print(f"os.chdir({here})") + os.chdir(f"{args.chroot_directory}") + os.symlink(target, basename) + os.chdir(here) + + +def chroot_mknod(line: str): + """ + Create a node with a given name, type, and major and minor number. + Currently character is the only node type supported. + """ + # We are passed a mknod terminal commmand. Need to parse it to make it work + # with os.mknod. + # :mknod + mknod_command = line.split() + + if not len(mknod_command) == 4: + raise ValueError(":mknod action expects 4 arguments!") + + node: str = mknod_command[0] + + # we only support creating character special devices + if mknod_command[1] != 'c': + print("Unrecognized mknod type; Valid type is 'c'") + sys.exit(-6) + + mode = 0o644 | stat.S_IFCHR + + # device major and minor numbers are created through os.makedev + major = mknod_command[2] + minor = mknod_command[3] + device = os.makedev(int(major), int(minor)) + + print(f"os.remove({args.chroot_directory}{node})") + print(f"os.mknod({args.chroot_directory}{node}, {mode}, {device})") + try: + os.remove(f"{args.chroot_directory}{node}") + except FileNotFoundError: + pass + + os.mknod(f"{args.chroot_directory}{node}", mode, device) + + +def chroot_cp(line: str): + """ + Copy a file from outside the chroot to inside the chroot, keeping the same + (relative) location. + """ + # :cp + print(f"shutil.copy({line}, {args.chroot_directory}{line})") + shutil.copy(line, f"{args.chroot_directory}{line}") + + +def chroot_cp_dir(line: str): + """ + Copy a directory from outside the chroot to inside the chroot, keeping the + same (relative) location. + """ + # :cp_dir + + print(f"shutil.copytree({line}, {args.chroot_directory}{line})") + # In Python v3.8+, shutil.copytree has a dirs_exist_ok keyword argument + # that prevents a FileExistsError when the destination directory exists. + # Unfortunately we are currently on v3.6.12, so we have to delete any + # directory that exists at that location. + if sys.version_info >= (3, 8): + shutil.copytree(line, f"{args.chroot_directory}{line}", + dirs_exist=True) + else: + if os.path.exists(f"{args.chroot_directory}{line}"): + shutil.rmtree(f"{args.chroot_directory}{line}") + shutil.copytree(line, f"{args.chroot_directory}{line}") + + +def chroot_chmod(line: str): + """ + Change the mode on a file in the chroot. + """ + # :chmod + + chmod_command = line.split() + + if not len(chmod_command) == 2: + raise ValueError(":chmod action expects 2 arguments!") + + mode = int(chmod_command[0], 8) + chmod_file = f"{args.chroot_directory}{chmod_command[1]}" + print(f"os.chmod({chmod_file}, {mode})") + os.chmod(chmod_file, mode) + + +def chroot_chmod_dir(line: str): + """ + Change the mode on a directory and everything inside it in the chroot. + """ + # :chmod_dir + chmod_command = line.split() + + if not len(chmod_command) == 2: + raise ValueError(":chmod_dir action expects 2 arguments!") + + mode = int(chmod_command[0], 8) + chmod_dir = f"{args.chroot_directory}{chmod_command[1]}" + for root, dirs, files in os.walk(f"{args.chroot_directory}{chmod_dir}"): + for directory in dirs: + print(f"os.chmod({os.path.join(root, directory)}, {int(mode, 8)})") + os.chmod(os.path.join(root, directory), int(mode, 8)) + for file in files: + print(f"os.chmod({os.path.join(root, file)}, {int(mode, 8)})") + os.chmod(os.path.join(root, file), int(mode, 8)) + + +def chroot_chown(line: str): + """ + Change ownership of a file in the chroot + """ + # :chown + chown_command = line.split() + + if not len(chown_command) == 2: + raise ValueError(":chown action expect 2 arguments!") + + user = chown_command[0] + chown_file = f"{args.chroot_directory}{chown_command[1]}" + print(f"os.chown({args.chroot_directory}{chown_file}, {user})") + os.chown(f"{args.chroot_directory}{chown_file, user}") + + +def chroot_chown_dir(line: str): + """ + Change ownership of a directory and everything inside it in the chroot. + """ + # :chown_dir + chown_command = line.split() + + if not len(chown_command) == 2: + raise ValueError(":chown_dir action expect 2 arguments!") + + user = chown_command[0] + chown_directory = f"{args.chroot_directory}{chown_command[1]}" + for root, dirs, files in os.walk(f"{args.chroot_directory}" + f"{chown_directory}"): + for directory in dirs: + print(f"os.chown({os.path.join(root, directory)}, {user})") + os.chown(os.path.join(root, directory), user) + for file in files: + print(f"os.chown({os.path.join(root, file)}, {user})") + os.chown(os.path.join(root, file), user) + + +def chroot_system(line: str): + """ + Executes a line from a .lst file using the system utility. + """ + # :system + line = line.rstrip() + print(f"system -i {line}") + subprocess.run(['system', '-i', line], check=True) + + +def chroot_sh(line: str): + """ + Executes a command from a .lst file using Pythons subprocess.run function. + """ + # :sh + print(line) + os.system(line) + subprocess.run(line.split(), check=True) + + +def chroot_file(chroot_lst: str): + """ + Parses and executes additional .lst files + """ + # :file + chroot_setup(chroot_lst) + + +def chroot_setup(chroot_lst): + """ + Parses and executes additional .lst files + """ + with open(chroot_lst, 'r') as lst_file: + action = None + # Iterate over all lines in the file, run actions where encountered + for line in lst_file: + if args.globals: + for key in globals_dictionary: + # sub placholders with values + line = re.sub(key, globals_dictionary[key], line) + + line = line.strip() + + # Skip blank lines and comments + if not line or line.startswith('#'): + continue + + # Determine action + elif line.startswith(':'): + # We enforce consistent function naming scheme for our action + # handlers. Each handler is prefixed with "chroot_" followed by + # the action name. The lst files prefix actions with ":" so we + # determine the action handler function name by prefixing it + # with "chroot_" and appending the action after the + # ":" from the line. + function = f"chroot_{line[1:]}" + try: + # use the globals dict to resolve action function handler + action = globals()[function] + except KeyError: + print(f"Skipping invalid action: {line}") + action = None + + # Execute action + else: + if action is not None: + action(line) + + +def initial_check(): + """ + Run an initial check on the environment, ensuring that we are running on + IBM i + """ + if os.uname().sysname == 'OS400': + print("**********************") + print("Live IBM i session (changes made).") + print("**********************") + + else: + print("**********************") + print("Not IBM i, no action is taken (debug flow purpose only).") + print("**********************") + sys.exit(-1) + + +def yum_install(package_list: list): + """ + Installs specified yum packages within the chroot. + """ + install_template = Template("/QOpenSys/pkgs/bin/yum $yes " + "--installroot=$chroot_dir install $packages") + auto_yes = '-y' if args.auto_yes else '' + packages = " ".join(package_list) + command = install_template.substitute(yes=auto_yes, + chroot_dir=args.chroot_directory, + packages=packages) + logging.debug(command) + subprocess.run(command.split(), check=True) + + +def input_yes(*args, **kwargs): + """ + Helper function used when auto yes mode is enabled. This function simple + returns 'y'. + """ + return 'y' + + +# +# MAIN +# +if __name__ == '__main__': + parser = argparse.ArgumentParser() + # positional arguments + parser.add_argument("chroot_directory", metavar="CHROOT_DIRECTORY", + help="The location of the chroot to be created") + + parser.add_argument("chroot_type", metavar="CHROOT_TYPE", nargs="*", + help="A list of .lst files to use for generating the " + "chroot") + # optional arguments + parser.add_argument("-g", dest='globals', nargs='*', metavar="", + help="Environment variables to set when creating the " + "chroot") + + parser.add_argument("-i", dest='yum_installs', nargs='*', metavar="", + help="A list of packages for yum to install in the " + "chroot") + + parser.add_argument("-y", dest='auto_yes', action='store_true', + help="Automatically answer yes to input prompts") + + parser.add_argument("-v", dest='verbose', action='store_true', + help="Print verbose output when running") + args = parser.parse_args() + parser.parse_args() + + CHROOT_DIR = "" + CHROOT_LIST = "" + SCRIPT_DIR = f"{os.path.dirname(os.path.realpath(__file__))}/config" + + print(SCRIPT_DIR) + + initial_check() + + # This is done because the arg may be a relative path name. + # Need to be done before any 'cd' + TEMP_DIR = os.path.realpath(args.chroot_directory) + print(TEMP_DIR) + + # if -v was passed, need to turn logging to DEBUG so all logging messages + # in the code are printed + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + logging.debug(f"Script directory: {SCRIPT_DIR}") + os.chdir(SCRIPT_DIR) + logging.debug(f"PWD: {os.getcwd()}") + + if not TEMP_DIR.startswith('/QOpenSys/'): + print("\nERROR: [CHROOT DIRECTORY] Must begin with /QOpenSys/...\n") + parser.print_help() + sys.exit(-2) + + CHROOT_DIR = TEMP_DIR + + if not os.path.isdir(CHROOT_DIR): + print(f"{CHROOT_DIR} does not exist.") + logging.debug(f"Creating directory at {CHROOT_DIR}") + os.mkdir(CHROOT_DIR) + else: + if args.auto_yes: + input = input_yes + + print(f"{CHROOT_DIR} directory already exists") + USER_INPUT = input("Would you like to continue chroot setup? [y/N]: ") + + if USER_INPUT != "y": + print("Bye") + sys.exit(-4) + + # Validate CHROOT TYPE Argument(s) + mylist = [] + + if not args.chroot_type: + # Only chroot_directory was provided to the script: + # Ask if minimal with includes chroot is desired + + if args.auto_yes: + input = input_yes + + USER_INPUT = input("Would you like to create a minimal chroot w/ " + f"includes @ {CHROOT_DIR}? [y/N]: ") + if USER_INPUT != "y": + print("Specify your desired [CHROOT TYPE]") + parser.print_help() + sys.exit(-5) + + mylist.append(f"{SCRIPT_DIR}/chroot_includes.lst") + mylist.append(f"{SCRIPT_DIR}/chroot_minimal.lst") + + else: + # If chroot types were specified, then use those + for arg in args.chroot_type: + print(f"Chroot type is: {arg}") + + CHROOT_LIST = arg + + # add chroot prefix if it doesn't exist + if not CHROOT_LIST.startswith('chroot'): + CHROOT_LIST = f"chroot_{CHROOT_LIST}" + + # check that the .lst file name was given + if not CHROOT_LIST.endswith('.lst'): + CHROOT_LIST = f"{CHROOT_LIST}.lst" + + # append file name to end of list + mylist.append(CHROOT_LIST) + + logging.debug(f"Number of elements in the list: {len(mylist)}") + logging.debug(f"List contents: {mylist}") + + # convert global variables list of "key=value" into a dictionary + globals_dictionary = {} + if args.globals: + for global_var in args.globals: + global_tokens = global_var.split('=') + globals_dictionary[global_tokens[0]] = global_tokens[1] + + for lst_file in mylist: + print("=====================================") + print(f"setting up based on {lst_file}") + print("=====================================") + chroot_setup(lst_file) + + os.makedirs(f"{args.chroot_directory}/QOpenSys/etc", exist_ok=True) + etc_profile = (f"{args.chroot_directory}/QOpenSys/etc/profile") + try: + with open(etc_profile, 'x') as profile: + print('Priming /QOpenSys/etc/profile') + print('PATH=/QOpenSys/pkgs/bin:$PATH', file=profile) + print('export PATH', file=profile) + print('OBJECT_MODE=64', file=profile) + print('export OBJECT_MODE', file=profile) + except FileExistsError: + logging.debug(f"{etc_profile} already exists") + + # perform installs if set + if args.yum_installs is not None: + yum_install(args.yum_installs) + + print(f""" +Done creating your chroot! + +To enter your chroot +Run: chroot {args.chroot_directory} /QOpenSys/usr/bin/sh + +To set up your PATH to pick up RPM packages once inside your chroot +Run: . /QOpenSys/etc/profile +""") diff --git a/config/chroot_includes.lst b/config/chroot_includes.lst index 4798366..cee412a 100644 --- a/config/chroot_includes.lst +++ b/config/chroot_includes.lst @@ -13,4 +13,3 @@ # :cp_dir /QOpenSys/usr/include - diff --git a/config/chroot_minimal.lst b/config/chroot_minimal.lst index adc9681..3d1887d 100644 --- a/config/chroot_minimal.lst +++ b/config/chroot_minimal.lst @@ -352,7 +352,7 @@ /QOpenSys/usr/lib/libld.a /QOpenSys/usr/lib/libm.a /QOpenSys/usr/lib/libnsl.a -/QOpenSys/usr/lib/libpmapi.a +# /QOpenSys/usr/lib/libpmapi.a /QOpenSys/usr/lib/libpthdebug.a /QOpenSys/usr/lib/libpthreads.a /QOpenSys/usr/lib/libpthreads_compat.a @@ -472,35 +472,35 @@ /QOpenSys/usr/bin /QOpenSys/bin /QOpenSys/usr/lib /QOpenSys/lib # alias -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/ -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/bg -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/cd -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/command -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/fc -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/fg -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/getopts -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/hash -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/jobs -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/read -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/type -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/ulimit -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/umask -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/unalias -/QOpenSys/usr/bin/alias /QOpenSys/usr/bin/wait +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/ +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/bg +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/cd +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/command +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/fc +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/fg +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/getopts +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/hash +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/jobs +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/read +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/type +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/ulimit +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/umask +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/unalias +# /QOpenSys/usr/bin/alias /QOpenSys/usr/bin/wait /QOpenSys/usr/bin/ksh /QOpenSys/usr/bin/sh /QOpenSys/usr/bin/rm /QOpenSys/usr/bin/rmdir # sbin to bin /QOpenSys/usr/sbin/chroot /QOpenSys/usr/bin/chroot /QOpenSys/usr/sbin/dump /QOpenSys/usr/bin/dump /QOpenSys/usr/sbin/trace /QOpenSys/usr/bin/trace -/QOpenSys/usr/sbin/trcoff /QOpenSys/usr/bin/trace -/QOpenSys/usr/sbin/trcon /QOpenSys/usr/bin/trace -/QOpenSys/usr/sbin/trcstop /QOpenSys/usr/bin/trace +# /QOpenSys/usr/sbin/trcoff /QOpenSys/usr/bin/trace +# /QOpenSys/usr/sbin/trcon /QOpenSys/usr/bin/trace +# /QOpenSys/usr/sbin/trcstop /QOpenSys/usr/bin/trace # individual libs /QOpenSys/usr/lib/libC128.a /QOpenSys/usr/lib/libC128_r.a /QOpenSys/usr/lib/libC.a /QOpenSys/usr/lib/libC_r.a /QOpenSys/usr/lib/libbsd.a /QOpenSys/usr/lib/libbsd_r.a -/QOpenSys/usr/lib/libc128.a /QOpenSys/usr/lib/libc_r.a +# /QOpenSys/usr/lib/libc128.a /QOpenSys/usr/lib/libc_r.a /QOpenSys/usr/lib/libc.a /QOpenSys/usr/lib/libc_r.a /QOpenSys/usr/lib/libc.a /QOpenSys/usr/lib/libc_t.a /QOpenSys/usr/lib/libxcurses.a /QOpenSys/usr/lib/libcurses.a @@ -563,7 +563,7 @@ /QOpenSys/usr/lib/libpthdebug.a /QOpenSys/usr/ccs/lib/libpthdebug.a /QOpenSys/usr/lib/libpthread.a /QOpenSys/usr/ccs/lib/libpthread.a /QOpenSys/usr/lib/libpthreads.a /QOpenSys/usr/ccs/lib/libpthreads.a -/QOpenSys/usr/lib/libpthreads_compat.a /QOpenSys/usr/ccs/lib/libpthreads_compat.a +/QOpenSys/usr/lib/libpthreads_compat.a /QOpenSys/usr/ccs/lib/bpthreads_compat.a /QOpenSys/usr/lib/librtl.a /QOpenSys/usr/ccs/lib/librtl.a /QOpenSys/usr/lib/libthread.a /QOpenSys/usr/ccs/lib/libthread.a /QOpenSys/usr/lib/libxcurses.a /QOpenSys/usr/ccs/lib/libxcurses.a @@ -611,12 +611,10 @@ # # run system command -# (system "CHGAUT OBJ('/QOpenSys/ranger/home/ranger') USER(RANGER) DTAAUT(*RWX) OBJAUT(*ALL) SUBTREE(*ALL)") +# (system "CHGAUT OBJ('/QOpenSys/ranger/home/ranger') USER(RANGER) DTAAUTRWX) # OBJAUT(*ALL) SUBTREE(*ALL)") # -- variable substitution example (mydir and myuser) -- -# > ./chroot_setup.sh chroot_system.lst /QOpenSys/ranger mydir=/QOpenSys/ranger myuser=RANGER +# > ./chroot_setup.sh chroot_system.lst /QOpenSys/ranger mydir=/QOpenSys/nger # myuser=RANGER # :system -# CHGAUT OBJ('/QOpenSys/ranger/home/ranger') USER(RANGER) DTAAUT(*RWX) OBJAUT(*ALL) SUBTREE(*ALL) -# CHGAUT OBJ('mydir/home/myuser') USER(myuser) DTAAUT(*RWX) OBJAUT(*ALL) SUBTREE(*ALL) - - +# CHGAUT OBJ('/QOpenSys/ranger/home/ranger') USER(RANGER) DTAAUT(*RWX) JAUT# (*ALL) SUBTREE(*ALL) +# CHGAUT OBJ('mydir/home/myuser') USER(myuser) DTAAUT(*RWX) OBJAUT(*ALL) SUBTREE# (*ALL) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..afd512c --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +flake8 >= 3.8.4