From 60024b124c4f7228779cab939c50f48f7f9cf362 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:00:23 -0400 Subject: [PATCH 01/36] add a deprefix helper and some unit tests Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 26 +++++++++++++++++++ .../test_config/test_environment.py | 17 ++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 collectoss/application/environment.py create mode 100644 tests/test_application/test_config/test_environment.py diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py new file mode 100644 index 000000000..4b737912f --- /dev/null +++ b/collectoss/application/environment.py @@ -0,0 +1,26 @@ + +def _deprefix(key: str, prefixes: list[str], separator = "_") -> str: + """Remove a prefix from the provided key + + + Args: + key (str): the key to remove the prefix from + prefixes (list[str]): the prefixes to look for + separator (str, optional): the separator between elements of the key to also remove (if they would otherwise be dangling). Defaults to "_". + + Returns: + str: The key value with the prefix removed if possible, otherwise returns the value of `key` + """ + unprefixed = None + for p in prefixes: + p = p.upper() + k = key.upper() + if k.startswith(p): + unprefixed = key[len(p):] + + if unprefixed.startswith(separator): + unprefixed = unprefixed[len(separator):] + return unprefixed + return key + + diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py new file mode 100644 index 000000000..bca34d92c --- /dev/null +++ b/tests/test_application/test_config/test_environment.py @@ -0,0 +1,17 @@ +from collectoss.application.environment import SystemEnv, _deprefix +import logging + +logger = logging.getLogger(__name__) + +prefixes = ["COLLECTOSS", "OTHER"] + +def test_env_deprefix(): + assert _deprefix("OTHER_DB", prefixes) == "DB" + assert _deprefix("COLLECTOSS_DB", prefixes) == "DB" + +def test_env_deprefix_default(): + assert _deprefix("SOME_DB", prefixes) == "SOME_DB" + assert _deprefix("THINGY_DB", prefixes) == "THINGY_DB" + +def test_env_deprefix_unprefixed(): + assert _deprefix("DB", prefixes) == "DB" From 67356974828b6bb34c725c1549a89b7211c7c886 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:16:24 -0400 Subject: [PATCH 02/36] refactor into a better prefix extraction helper Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 21 +++++++++---------- .../test_config/test_environment.py | 19 +++++++++-------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 4b737912f..4bb02b631 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -1,7 +1,7 @@ +from typing import Optional -def _deprefix(key: str, prefixes: list[str], separator = "_") -> str: - """Remove a prefix from the provided key - +def extract_prefix(key: str, prefixes: list[str], separator = "_") -> Optional[str]: + """Detect and return the prefix present on the provided key Args: key (str): the key to remove the prefix from @@ -9,18 +9,17 @@ def _deprefix(key: str, prefixes: list[str], separator = "_") -> str: separator (str, optional): the separator between elements of the key to also remove (if they would otherwise be dangling). Defaults to "_". Returns: - str: The key value with the prefix removed if possible, otherwise returns the value of `key` + str: The detected prefix (including any separators) if any, otherwise None """ - unprefixed = None + prefix_len = 0 for p in prefixes: p = p.upper() k = key.upper() if k.startswith(p): - unprefixed = key[len(p):] - - if unprefixed.startswith(separator): - unprefixed = unprefixed[len(separator):] - return unprefixed - return key + prefix_len += len(p) + if k[prefix_len] == separator: + prefix_len += len(separator) + return key[0:prefix_len] + return None diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index bca34d92c..d3248412c 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -1,17 +1,18 @@ -from collectoss.application.environment import SystemEnv, _deprefix +from collectoss.application.environment import SystemEnv, extract_prefix import logging logger = logging.getLogger(__name__) prefixes = ["COLLECTOSS", "OTHER"] -def test_env_deprefix(): - assert _deprefix("OTHER_DB", prefixes) == "DB" - assert _deprefix("COLLECTOSS_DB", prefixes) == "DB" +def test_env_extract_prefix(): + assert extract_prefix("OTHER_DB", prefixes) == "OTHER_" + assert extract_prefix("COLLECTOSS_DB", prefixes) == "COLLECTOSS_" -def test_env_deprefix_default(): - assert _deprefix("SOME_DB", prefixes) == "SOME_DB" - assert _deprefix("THINGY_DB", prefixes) == "THINGY_DB" +def test_env_extract_prefix_default(): + assert extract_prefix("SOME_DB", prefixes) is None + assert extract_prefix("THINGY_DB", prefixes) is None -def test_env_deprefix_unprefixed(): - assert _deprefix("DB", prefixes) == "DB" + +def test_env_extract_prefix_unprefixed(): + assert extract_prefix("DB", prefixes) is None From a5475003db1a3349e7529b50b29b8d60552aa26c Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:21:42 -0400 Subject: [PATCH 03/36] add first pass SystemEnv Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 4bb02b631..3bf9fdb3e 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -1,4 +1,9 @@ from typing import Optional +import os +import warnings +import logging + +logger = logging.getLogger(__name__) def extract_prefix(key: str, prefixes: list[str], separator = "_") -> Optional[str]: """Detect and return the prefix present on the provided key @@ -23,3 +28,35 @@ def extract_prefix(key: str, prefixes: list[str], separator = "_") -> Optional[s return key[0:prefix_len] return None + +class SystemEnv: + """Centralized environment variable access + Built for enabling migration of environment variable names + """ + + _prefixes = ["COLLECTOSS", "AUGUR"] + _warn_prefixes = ["AUGUR"] + _separator = "_" + + @classmethod + def get(cls, key: str, default = None) -> Optional[str]: + # extract the suffix so we can try multiple prefixes + canonical_prefix = extract_prefix(key, cls._prefixes, cls._separator) + suffix = key[len(canonical_prefix):] if canonical_prefix is not None else key + # check prefixes in order and use the first one that has a value + for p in cls._prefixes: + check_key = f"{p}{cls._separator}{suffix}" + value = os.getenv(check_key, None) + + if value is not None: + # emit a warning if configured + if p in cls._warn_prefixes: + msg = ( + f"Environment variable '{check_key}' is deprecated. " + f"Use '{key}' instead. Automatic recovery may be removed in a future version" + ) + logger.warning(msg) + warnings.warn(msg, DeprecationWarning, stacklevel=2) + + return value + return default From a0b3e200f638d1d9174ebb5bd0e66d5a4c535536 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:39:55 -0400 Subject: [PATCH 04/36] allow different prefixes to be passed in for testing purposes Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 3bf9fdb3e..7cadc0dfc 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -39,12 +39,12 @@ class SystemEnv: _separator = "_" @classmethod - def get(cls, key: str, default = None) -> Optional[str]: + def get(cls, key: str, default = None, prefixes = _prefixes) -> Optional[str]: # extract the suffix so we can try multiple prefixes - canonical_prefix = extract_prefix(key, cls._prefixes, cls._separator) + canonical_prefix = extract_prefix(key, prefixes, cls._separator) suffix = key[len(canonical_prefix):] if canonical_prefix is not None else key # check prefixes in order and use the first one that has a value - for p in cls._prefixes: + for p in prefixes: check_key = f"{p}{cls._separator}{suffix}" value = os.getenv(check_key, None) From 657cc533c6962b473efaf5c1029d223620485294 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:40:08 -0400 Subject: [PATCH 05/36] update deprecation message Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 7cadc0dfc..ae9d81bb7 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -53,7 +53,7 @@ def get(cls, key: str, default = None, prefixes = _prefixes) -> Optional[str]: if p in cls._warn_prefixes: msg = ( f"Environment variable '{check_key}' is deprecated. " - f"Use '{key}' instead. Automatic recovery may be removed in a future version" + f"Use '{key}' instead. This automatic recovery may become a failure in a future version " ) logger.warning(msg) warnings.warn(msg, DeprecationWarning, stacklevel=2) From 578c4a331096ed11f98bacc98bc0166d9ab1586f Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 14:42:44 -0400 Subject: [PATCH 06/36] basic functionality unit tests Signed-off-by: Adrian Edwards --- .../test_config/test_environment.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index d3248412c..11c1e65aa 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -1,5 +1,6 @@ from collectoss.application.environment import SystemEnv, extract_prefix import logging +import os logger = logging.getLogger(__name__) @@ -16,3 +17,23 @@ def test_env_extract_prefix_default(): def test_env_extract_prefix_unprefixed(): assert extract_prefix("DB", prefixes) is None + +def test_fetching_env(): + # plain + os.environ["COLLECTOSS_NAME"] = "A" + assert SystemEnv.get("COLLECTOSS_NAME") == "A" + + # fallback handling + os.environ["OTHER_THING"] = "B" + assert SystemEnv.get("COLLECTOSS_THING", None, prefixes) == "B" + + # cleanup + del os.environ["COLLECTOSS_NAME"] + del os.environ["OTHER_THING"] + +def test_fetching_env_no_value(): + assert SystemEnv.get("COLLECTOSS_MISSING", None, prefixes) is None + +def test_fetching_env_default(): + assert SystemEnv.get("COLLECTOSS_DEFAULT", "SOME", prefixes) == "SOME" + From ac8fceffc04fdf42060d76be273f942ca57238a4 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:00:36 -0400 Subject: [PATCH 07/36] handle cases with no known prefix to avoid breaking stuff Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 4 ++++ tests/test_application/test_config/test_environment.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index ae9d81bb7..c2eca8b15 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -59,4 +59,8 @@ def get(cls, key: str, default = None, prefixes = _prefixes) -> Optional[str]: warnings.warn(msg, DeprecationWarning, stacklevel=2) return value + + if not canonical_prefix: + return os.getenv(key, default) + return default diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index 11c1e65aa..3b31bf950 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -37,3 +37,8 @@ def test_fetching_env_no_value(): def test_fetching_env_default(): assert SystemEnv.get("COLLECTOSS_DEFAULT", "SOME", prefixes) == "SOME" +def test_no_known_prefix(): + # fallback handling + os.environ["THING"] = "C" + assert SystemEnv.get("THING", None, prefixes) == "C" + From 6aa89a21d5c6b398008c913bb289267b63959cf0 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:10:22 -0400 Subject: [PATCH 08/36] factor out bool fetching class to deduplicate logic Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 5 ++++ .../test_config/test_environment.py | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index c2eca8b15..33bb0a7f5 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -64,3 +64,8 @@ def get(cls, key: str, default = None, prefixes = _prefixes) -> Optional[str]: return os.getenv(key, default) return default + + @classmethod + def get_bool(cls, key:str, default: bool, prefixes = _prefixes) -> bool: + raw_val = cls.get(key, None, prefixes) + return raw_val.lower() in ('true', '1', 't', 'y', 'yes') if raw_val else default diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index 3b31bf950..aa00bcb41 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -42,3 +42,32 @@ def test_no_known_prefix(): os.environ["THING"] = "C" assert SystemEnv.get("THING", None, prefixes) == "C" + +def test_get_bool_trues(): + + cases = ["1", "true", "True", "TRUE", "y", "Y", "yes", "Yes"] + + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == True + del os.environ["OTHER_BOOL"] + +def test_get_bool_falses(): + + cases = ["0", "false", "False", "FALSE", "n", "N", "no", "No"] + + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", True, prefixes) == False + del os.environ["OTHER_BOOL"] + +def test_get_bool_default(): + + cases = ["?", "maybe", "Stuff", "333"] + + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == False + del os.environ["OTHER_BOOL"] + + From 5d33715d1d2f4cf5fc98e19bc946f7b27f710c00 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:20:12 -0400 Subject: [PATCH 09/36] Test to make sure we can potentially migrate the AUGUR_ vars in the code slowly over time Signed-off-by: Adrian Edwards --- tests/test_application/test_config/test_environment.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index aa00bcb41..6b62f2ec9 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -31,6 +31,13 @@ def test_fetching_env(): del os.environ["COLLECTOSS_NAME"] del os.environ["OTHER_THING"] +def test_fetching_env_backwards(): + os.environ["COLLECTOSS_NAME"] = "A" + assert SystemEnv.get("OTHER_NAME", None, prefixes) == "A" + + # cleanup + del os.environ["COLLECTOSS_NAME"] + def test_fetching_env_no_value(): assert SystemEnv.get("COLLECTOSS_MISSING", None, prefixes) is None From 1e07a3a66c9fb9003bbb0eb2bd88ee1a4fd691a3 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:20:28 -0400 Subject: [PATCH 10/36] replace all references to os.getenv with new class Signed-off-by: Adrian Edwards --- collectoss/api/gunicorn_conf.py | 8 +++++--- collectoss/application/cli/__init__.py | 8 +++++--- collectoss/application/cli/api.py | 4 +++- collectoss/application/cli/backend.py | 11 ++++++----- collectoss/application/cli/collection.py | 3 ++- collectoss/application/cli/db.py | 5 +++-- collectoss/application/config.py | 6 ++++-- collectoss/tasks/git/dependency_tasks/core.py | 3 ++- collectoss/tasks/git/scc_value_tasks/core.py | 3 ++- .../git/util/facade_worker/facade_worker/config.py | 4 +++- 10 files changed, 35 insertions(+), 20 deletions(-) diff --git a/collectoss/api/gunicorn_conf.py b/collectoss/api/gunicorn_conf.py index 22c11231a..ee7797471 100644 --- a/collectoss/api/gunicorn_conf.py +++ b/collectoss/api/gunicorn_conf.py @@ -7,6 +7,7 @@ from collectoss.application.db.lib import get_value from collectoss.application.db import dispose_database_engine +from collectoss.application.environment import SystemEnv logger = logging.getLogger(__name__) @@ -20,8 +21,8 @@ workers = multiprocessing.cpu_count() * 2 + 1 umask = 0o007 reload = True - -is_dev = os.getenv("AUGUR_DEV", 'False').lower() in ('true', '1', 't', 'y', 'yes') +# this satisfies the type checker +is_dev = SystemEnv.get_bool("AUGUR_DEV", False) if is_dev: @@ -40,7 +41,8 @@ # set the log location for gunicorn logs_directory = get_value('Logging', 'logs_directory') -is_docker = os.getenv("AUGUR_DOCKER_DEPLOY", 'False').lower() in ('true', '1', 't', 'y', 'yes') +# this syntax satisfies the type checker +is_docker = SystemEnv.get_bool("AUGUR_DOCKER_DEPLOY", False) accesslog = f"{logs_directory}/gunicorn.log" errorlog = f"{logs_directory}/gunicorn.log" diff --git a/collectoss/application/cli/__init__.py b/collectoss/application/cli/__init__.py index 8081d6a8e..b398614e2 100644 --- a/collectoss/application/cli/__init__.py +++ b/collectoss/application/cli/__init__.py @@ -10,7 +10,9 @@ from collectoss.application.db.engine import DatabaseEngine from collectoss.application.db import get_engine, dispose_database_engine -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError +from collectoss.application.environment import SystemEnv + def check_connectivity(urls=["http://chaoss.community", "http://github.com", "http://gitlab.com"], timeout=10.0): @@ -65,11 +67,11 @@ def new_func(ctx, *args, **kwargs): return ctx.invoke(function_db_connection, *args, **kwargs) except OperationalError as e: - db_environment_var = os.getenv("AUGUR_DB") + db_environment_var = SystemEnv.get("AUGUR_DB") # determine the location to print in error string if db_environment_var: - location = f"the AUGUR_DB environment variable\nAUGUR_DB={os.getenv('AUGUR_DB')}" + location = f"the AUGUR_DB environment variable\nAUGUR_DB={SystemEnv.get('AUGUR_DB')}" else: with open("db.config.json", 'r') as f: db_config = json.load(f) diff --git a/collectoss/application/cli/api.py b/collectoss/application/cli/api.py index a8bb9e53b..70fe3a6a1 100644 --- a/collectoss/application/cli/api.py +++ b/collectoss/application/cli/api.py @@ -17,6 +17,8 @@ from collectoss.application.cli import test_connection, test_db_connection, with_database, DatabaseContext from collectoss.application.cli._cli_util import _broadcast_signal_to_processes, raise_open_file_limit, clear_redis_caches, clear_rabbitmq_messages from collectoss.application.db.lib import get_value +from collectoss.application.environment import SystemEnv + logger = SystemLogger("collectoss", reset_logfiles=False).get_logger() @@ -142,7 +144,7 @@ def get_api_processes(): def is_api_process(process): command = ''.join(process.info['cmdline'][:]).lower() - if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command: + if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command: if process.pid != os.getpid(): diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index a07ddf198..093c90d54 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -15,6 +15,7 @@ import requests from redis.exceptions import ConnectionError as RedisConnectionError +from collectoss.application.environment import SystemEnv from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records from collectoss.tasks.git.facade_tasks import clone_repos from collectoss.tasks.github.contributors import process_contributors @@ -31,7 +32,7 @@ from keyman.KeyClient import KeyClient, KeyPublisher -reset_logs = os.getenv("AUGUR_RESET_LOGS", 'True').lower() in ('true', '1', 't', 'y', 'yes') +reset_logs = SystemEnv.get_bool("AUGUR_RESET_LOGS", True) logger = SystemLogger("collectoss", reset_logfiles=reset_logs).get_logger() @@ -130,7 +131,7 @@ def start(ctx, disable_collection, development, pidfile, port): processes = start_celery_worker_processes((core_worker_count, secondary_worker_count, facade_worker_count), disable_collection) manager.processes = processes - celery_beat_schedule_db = os.getenv("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db") + celery_beat_schedule_db = SystemEnv.get("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db") if os.path.exists(celery_beat_schedule_db): logger.info("Deleting old task schedule") os.remove(celery_beat_schedule_db) @@ -355,10 +356,10 @@ def export_env(config): Exports your GitHub key and database credentials """ - export_file = open(os.getenv('AUGUR_EXPORT_FILE', 'collectoss_export_env.sh'), 'w+') + export_file = open(SystemEnv.get('AUGUR_EXPORT_FILE') or 'collectoss_export_env.sh', 'w+') export_file.write('#!/bin/bash') export_file.write('\n') - env_file = open(os.getenv('AUGUR_ENV_FILE', 'docker_env.txt'), 'w+') + env_file = open(SystemEnv.get('AUGUR_ENV_FILE') or 'docker_env.txt', 'w+') for env_var in config.get_env_config().items(): if "LOG" not in env_var[0]: @@ -403,7 +404,7 @@ def get_backend_processes(): for process in psutil.process_iter(['cmdline', 'name', 'environ']): if process.info['cmdline'] is not None and process.info['environ'] is not None: try: - if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower(): + if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower(): if process.pid != os.getpid(): process_list.append(process) except (KeyError, FileNotFoundError): diff --git a/collectoss/application/cli/collection.py b/collectoss/application/cli/collection.py index b1a93ce80..3b4f71dc7 100644 --- a/collectoss/application/cli/collection.py +++ b/collectoss/application/cli/collection.py @@ -14,6 +14,7 @@ import traceback import sqlalchemy as s +from collectoss.application.environment import SystemEnv from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records from collectoss.tasks.git.facade_tasks import clone_repos from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler @@ -237,7 +238,7 @@ def get_collection_processes(): def is_collection_process(process): command = ''.join(process.info['cmdline'][:]).lower() - if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command: + if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command: if process.pid != os.getpid(): if "collectossbackendcollection" in command or "celery_app.celery_appbeat" in command: diff --git a/collectoss/application/cli/db.py b/collectoss/application/cli/db.py index 1a790e3c0..a93668b3e 100644 --- a/collectoss/application/cli/db.py +++ b/collectoss/application/cli/db.py @@ -28,6 +28,7 @@ process_repo_csv, process_repo_group_csv, ) +from collectoss.application.environment import SystemEnv logger = logging.getLogger(__name__) @@ -379,7 +380,7 @@ def get_api_key(ctx): short_help="Check the ~/.pgpass file for CollectOSS's database credentials", ) def check_pgpass(): - db_environment_var = getenv("AUGUR_DB") + db_environment_var = SystemEnv.get("AUGUR_DB") if db_environment_var: # gets the user, passowrd, host, port, and database_name out of environment variable # assumes database string of structure //:@:/ @@ -495,7 +496,7 @@ def run_psql_command_in_database(target_type, target): logger.error("Invalid target type. Exiting...") exit(1) - db_environment_var = getenv("AUGUR_DB") + db_environment_var = SystemEnv.get("AUGUR_DB") # db_json_file_location = os.getcwd() + "/db.config.json" # db_json_exists = os.path.exists(db_json_file_location) diff --git a/collectoss/application/config.py b/collectoss/application/config.py index 56e6c57ae..051235323 100644 --- a/collectoss/application/config.py +++ b/collectoss/application/config.py @@ -7,6 +7,8 @@ from collectoss.application.db.models import Config from collectoss.application.db.util import execute_session_query, convert_type_of_value from pathlib import Path +from collectoss.application.environment import SystemEnv + import logging def get_development_flag_from_config(): @@ -27,7 +29,7 @@ def get_development_flag_from_config(): return flag def get_development_flag(): - return os.getenv("AUGUR_DEV") or get_development_flag_from_config() or False + return SystemEnv.get("AUGUR_DEV") or get_development_flag_from_config() or False def redact_setting_value(section_name, setting_name, value): value_redacted = value if section_name != "Keys" else "REDACTED" @@ -167,7 +169,7 @@ def __init__(self, logger, session: DatabaseSession, config_sources: list = None JsonConfig(default_config, logger) ] - config_dir = Path(os.getenv("CONFIG_DATADIR", "./")) + config_dir = Path(SystemEnv.get("CONFIG_DATADIR") or "./") config_path = config_dir.joinpath("augur.json") if config_path.exists(): config_sources.append(JsonConfig(json.loads(config_path.read_text(encoding="UTF-8")), logger)) diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index a9e74b4e1..0b713de93 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -2,6 +2,7 @@ import os from collectoss.application.db.models import * from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value, get_session +from collectoss.application.environment import SystemEnv from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler from collectoss.tasks.git.dependency_tasks.dependency_util import dependency_calculator as dep_calc from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call @@ -79,7 +80,7 @@ def generate_scorecard(logger, repo_git): command = '--repo=' + path #this is path where our scorecard project is located - path_to_scorecard = os.getenv('SCORECARD_DIR', os.environ['HOME'] + '/scorecard') + path_to_scorecard = SystemEnv.get('SCORECARD_DIR', os.environ['HOME'] + '/scorecard') #setting the environmental variable which is required by scorecard diff --git a/collectoss/tasks/git/scc_value_tasks/core.py b/collectoss/tasks/git/scc_value_tasks/core.py index 7c9e0bafd..a526af990 100644 --- a/collectoss/tasks/git/scc_value_tasks/core.py +++ b/collectoss/tasks/git/scc_value_tasks/core.py @@ -2,6 +2,7 @@ import os from collectoss.application.db.models import * from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value +from collectoss.application.environment import SystemEnv from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path @@ -20,7 +21,7 @@ def value_model(logger,repo_git): logger.info(f"Repo ID: {repo_id}, Path: {path}") logger.info('Running scc...') - path_to_scc = os.getenv('SCC_DIR', os.environ['HOME'] + '/scc') + path_to_scc = SystemEnv.get('SCC_DIR', (SystemEnv.get('HOME') or "~") + '/scc') required_output = parse_json_from_subprocess_call(logger,['./scc', '-f','json','--by-file', path], cwd=path_to_scc) diff --git a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py index 7da6495bd..9db7d8866 100644 --- a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py +++ b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py @@ -40,11 +40,13 @@ from collectoss.application.db.lib import execute_sql from logging import Logger +from collectoss.application.environment import SystemEnv + logger = logging.getLogger(__name__) def get_database_args_from_env(): - db_str = os.getenv("AUGUR_DB") + db_str = SystemEnv.get("AUGUR_DB") try: db_json_file_location = os.getcwd() + "/db.config.json" except FileNotFoundError: From af281ecb08a1b16f540609cc6a22f28d36979a37 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:20:48 -0400 Subject: [PATCH 11/36] deprecate older Environment class that is buried in the module tree Signed-off-by: Adrian Edwards --- collectoss/api/view/server/Environment.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/collectoss/api/view/server/Environment.py b/collectoss/api/view/server/Environment.py index 76b8207ca..4d35bc70a 100644 --- a/collectoss/api/view/server/Environment.py +++ b/collectoss/api/view/server/Environment.py @@ -1,4 +1,5 @@ import os +from typing_extensions import deprecated class Environment: """ @@ -7,16 +8,19 @@ class Environment: with subscript notation without needing to deal with the particularities of non-existent values. """ + @deprecated("use collectoss.application.environment.SystemEnv instead") def __init__(self, **kwargs): for (key, value) in kwargs.items(): self[key] = value + @deprecated("use collectoss.application.environment.SystemEnv instead") def setdefault(self, key, value): if not self[key]: self[key] = value return value return self[key] + @deprecated("use collectoss.application.environment.SystemEnv instead") def setall(self, **kwargs): result = {} for (key, value) in kwargs.items(): @@ -24,6 +28,7 @@ def setall(self, **kwargs): result[key] = self[key] self[key] = value + @deprecated("use collectoss.application.environment.SystemEnv instead") def getany(self, *args): result = {} for arg in args: @@ -31,6 +36,7 @@ def getany(self, *args): result[arg] = self[arg] return result + @deprecated("use collectoss.application.environment.SystemEnv instead") def as_type(self, type, key): if self[key]: return type(self[key]) From 01c2fad0f34f5b1c40acaccfac6334cc39d058d6 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:27:24 -0400 Subject: [PATCH 12/36] get_bool docstring Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 33bb0a7f5..462386f72 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -67,5 +67,7 @@ def get(cls, key: str, default = None, prefixes = _prefixes) -> Optional[str]: @classmethod def get_bool(cls, key:str, default: bool, prefixes = _prefixes) -> bool: + """gets a value from the environment and cast it to a boolean + """ raw_val = cls.get(key, None, prefixes) return raw_val.lower() in ('true', '1', 't', 'y', 'yes') if raw_val else default From f52626b6a02b93f0e494b8cd6ab73697a26f321b Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:27:34 -0400 Subject: [PATCH 13/36] basic setter Signed-off-by: Adrian Edwards --- collectoss/application/environment.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 462386f72..3a28c12a9 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -71,3 +71,10 @@ def get_bool(cls, key:str, default: bool, prefixes = _prefixes) -> bool: """ raw_val = cls.get(key, None, prefixes) return raw_val.lower() in ('true', '1', 't', 'y', 'yes') if raw_val else default + + @classmethod + def set(cls, key: str, value: str, overwrite=True) -> None: + if os.getenv(key) is not None and not overwrite: + return + + os.environ[key] = value \ No newline at end of file From 35845f1f55f64cf32e7e62f9a554e2260827ac59 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:32:05 -0400 Subject: [PATCH 14/36] replace references to os.environ[] with new class Signed-off-by: Adrian Edwards --- collectoss/api/routes/auggie.py | 4 +++- collectoss/application/cli/api.py | 2 +- collectoss/application/cli/backend.py | 8 ++++---- collectoss/application/cli/collection.py | 2 +- collectoss/tasks/git/dependency_tasks/core.py | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/collectoss/api/routes/auggie.py b/collectoss/api/routes/auggie.py index 18642498f..f02122ae3 100644 --- a/collectoss/api/routes/auggie.py +++ b/collectoss/api/routes/auggie.py @@ -14,6 +14,8 @@ import requests import slack +from collectoss.application.environment import SystemEnv + from ..server import app @@ -326,7 +328,7 @@ def slack_login(): print("slack_login") r = requests.get( - url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={os.environ["AUGGIE_CLIENT_ID"]}&client_secret={os.environ["AUGGIE_CLIENT_SECRET"]}&redirect_uri=http%3A%2F%2Flocalhost%3A8080') + url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={SystemEnv.get("AUGGIE_CLIENT_ID")}&client_secret={SystemEnv.get("AUGGIE_CLIENT_SECRET")}&redirect_uri=http%3A%2F%2Flocalhost%3A8080') data = r.json() if (data["ok"]): diff --git a/collectoss/application/cli/api.py b/collectoss/application/cli/api.py index 70fe3a6a1..e3e4a5a55 100644 --- a/collectoss/application/cli/api.py +++ b/collectoss/application/cli/api.py @@ -48,7 +48,7 @@ def start(ctx, development, port): raise e if development: - os.environ["AUGUR_DEV"] = "1" + SystemEnv.set("AUGUR_DEV", "1") logger.info("Starting in development mode") try: diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index 093c90d54..b2a4ed038 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -72,10 +72,10 @@ def start(ctx, disable_collection, development, pidfile, port): raise e if development: - os.environ["AUGUR_DEV"] = "1" + SystemEnv.set("AUGUR_DEV", "1") logger.info("Starting in development mode") - os.environ["AUGUR_PIDFILE"] = pidfile + SystemEnv.set("AUGUR_PIDFILE", pidfile) try: gunicorn_location = os.getcwd() + "/collectoss/api/gunicorn_conf.py" @@ -87,10 +87,10 @@ def start(ctx, disable_collection, development, pidfile, port): if not port: port = get_value("Server", "port") - os.environ["AUGUR_PORT"] = str(port) + SystemEnv.set("AUGUR_PORT", str(port)) if disable_collection: - os.environ["AUGUR_DISABLE_COLLECTION"] = "1" + SystemEnv.set("AUGUR_DISABLE_COLLECTION", "1") core_worker_count = get_value("Celery", 'core_worker_count') secondary_worker_count = get_value("Celery", 'secondary_worker_count') diff --git a/collectoss/application/cli/collection.py b/collectoss/application/cli/collection.py index 3b4f71dc7..136024984 100644 --- a/collectoss/application/cli/collection.py +++ b/collectoss/application/cli/collection.py @@ -76,7 +76,7 @@ def start(ctx, development): keypub.publish(key, "gitlab_rest") if development: - os.environ["AUGUR_DEV"] = "1" + SystemEnv.set("AUGUR_DEV", "1") logger.info("Starting in development mode") core_worker_count = get_value("Celery", 'core_worker_count') diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index 0b713de93..0e5c55b9e 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -80,14 +80,14 @@ def generate_scorecard(logger, repo_git): command = '--repo=' + path #this is path where our scorecard project is located - path_to_scorecard = SystemEnv.get('SCORECARD_DIR', os.environ['HOME'] + '/scorecard') + path_to_scorecard = SystemEnv.get('SCORECARD_DIR', (SystemEnv.get('HOME') or "~") + '/scorecard') #setting the environmental variable which is required by scorecard with get_session() as session: #key_handler = GithubRandomKeyAuth(logger) key_handler = GithubApiKeyHandler(logger) - os.environ['GITHUB_AUTH_TOKEN'] = key_handler.get_random_key() + SystemEnv.set('GITHUB_AUTH_TOKEN', key_handler.get_random_key()) # This seems outdated #setting the environmental variable which is required by scorecard From ed673bfaf1e9208abd20ea4a9cd4eb8bd10f3553 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 15:32:24 -0400 Subject: [PATCH 15/36] remove some redundant wrapping code Signed-off-by: Adrian Edwards --- collectoss/tasks/git/dependency_tasks/core.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index 0e5c55b9e..21f24246a 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -83,16 +83,8 @@ def generate_scorecard(logger, repo_git): path_to_scorecard = SystemEnv.get('SCORECARD_DIR', (SystemEnv.get('HOME') or "~") + '/scorecard') #setting the environmental variable which is required by scorecard - - with get_session() as session: - #key_handler = GithubRandomKeyAuth(logger) - key_handler = GithubApiKeyHandler(logger) - SystemEnv.set('GITHUB_AUTH_TOKEN', key_handler.get_random_key()) - - # This seems outdated - #setting the environmental variable which is required by scorecard - #key_handler = GithubApiKeyHandler(session, session.logger) - #os.environ['GITHUB_AUTH_TOKEN'] = key_handler.get_random_key() + key_handler = GithubApiKeyHandler(logger) + SystemEnv.set('GITHUB_AUTH_TOKEN', key_handler.get_random_key()) try: required_output = parse_json_from_subprocess_call(logger,['./scorecard', command, '--format=json'],cwd=path_to_scorecard) From c72520e3b85a85e43f3eab0ef07df5e02591373e Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 16:28:10 -0400 Subject: [PATCH 16/36] replace references to os.environ.get Signed-off-by: Adrian Edwards --- collectoss/api/routes/auggie.py | 6 +++--- collectoss/application/cli/api.py | 2 +- collectoss/application/cli/backend.py | 4 ++-- collectoss/application/cli/collection.py | 2 +- collectoss/tasks/init/celery_app.py | 2 +- collectoss/tasks/start_tasks.py | 5 +++-- keyman/Orchestrator.py | 5 +++-- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/collectoss/api/routes/auggie.py b/collectoss/api/routes/auggie.py index f02122ae3..6d036045a 100644 --- a/collectoss/api/routes/auggie.py +++ b/collectoss/api/routes/auggie.py @@ -254,7 +254,7 @@ def get_auggie_user(): # return Response(response=response, status=200, mimetype="application/json") ## From Method profile_name = 'collectoss' - if os.environ.get('AUGUR_IS_PROD'): + if SystemEnv.get('AUGUR_IS_PROD'): profile_name = 'default' client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') response = client.get_item( @@ -280,7 +280,7 @@ def update_auggie_user_tracking(): # return Response(response=response, status=200, mimetype="application/json") ## From Method profile_name = 'collectoss' - if os.environ.get('AUGUR_IS_PROD'): + if SystemEnv.get('AUGUR_IS_PROD'): profile_name = 'default' client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') response = client.update_item( @@ -342,7 +342,7 @@ def slack_login(): email = user_response["user"]["email"] profile_name = 'collectoss' - if os.environ.get('AUGUR_IS_PROD'): + if SystemEnv.get('AUGUR_IS_PROD'): profile_name = 'default' print("Making Boto3 Session") client = boto3.Session(region_name='us-east-1', diff --git a/collectoss/application/cli/api.py b/collectoss/application/cli/api.py index e3e4a5a55..4f7077a78 100644 --- a/collectoss/application/cli/api.py +++ b/collectoss/application/cli/api.py @@ -38,7 +38,7 @@ def start(ctx, development, port): """Start CollectOSS's backend server.""" try: - if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index b2a4ed038..d4586083b 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -62,7 +62,7 @@ def start(ctx, disable_collection, development, pidfile, port): signal.signal(signal.SIGINT, manager.shutdown_signal_handler) try: - if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( @@ -145,7 +145,7 @@ def start(ctx, disable_collection, development, pidfile, port): manager.keypub = keypub if not disable_collection: - if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": orchestrator = subprocess.Popen("python keyman/Orchestrator.py".split()) # Wait for orchestrator startup diff --git a/collectoss/application/cli/collection.py b/collectoss/application/cli/collection.py index 136024984..5127a8d17 100644 --- a/collectoss/application/cli/collection.py +++ b/collectoss/application/cli/collection.py @@ -46,7 +46,7 @@ def start(ctx, development): """Start CollectOSS's backend server.""" try: - if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( diff --git a/collectoss/tasks/init/celery_app.py b/collectoss/tasks/init/celery_app.py index e14230f99..a33e1e961 100644 --- a/collectoss/tasks/init/celery_app.py +++ b/collectoss/tasks/init/celery_app.py @@ -63,7 +63,7 @@ tasks = start_tasks + github_tasks + gitlab_tasks + git_tasks + materialized_view_tasks + frontend_tasks -if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": +if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": tasks += data_analysis_tasks redis_db_number, redis_conn_string = get_redis_conn_values() diff --git a/collectoss/tasks/start_tasks.py b/collectoss/tasks/start_tasks.py index 8e130f926..e7f86b73b 100644 --- a/collectoss/tasks/start_tasks.py +++ b/collectoss/tasks/start_tasks.py @@ -14,7 +14,8 @@ from collectoss.tasks.github.pull_requests.tasks import * from collectoss.tasks.github.repo_info.tasks import * from collectoss.tasks.github.releases.tasks import * -if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1": +from collectoss.application.environment import SystemEnv +if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": from collectoss.tasks.data_analysis import * from collectoss.tasks.github.detect_move.tasks import detect_github_repo_move_core, detect_github_repo_move_secondary from collectoss.tasks.github.releases.tasks import collect_releases @@ -38,7 +39,7 @@ from collectoss.application.db.lib import execute_sql, get_session from collectoss.application.config import SystemConfig -RUNNING_DOCKER = os.environ.get('AUGUR_DOCKER_DEPLOY') == "1" +RUNNING_DOCKER = SystemEnv.get('AUGUR_DOCKER_DEPLOY') == "1" CELERY_GROUP_TYPE = type(group()) CELERY_CHAIN_TYPE = type(chain()) diff --git a/keyman/Orchestrator.py b/keyman/Orchestrator.py index 71cfae8bb..d93a1f064 100644 --- a/keyman/Orchestrator.py +++ b/keyman/Orchestrator.py @@ -4,15 +4,16 @@ import time from keyman.KeyOrchestrationAPI import spec, WaitKeyTimeout, InvalidRequest +from collectoss.application.environment import SystemEnv -if os.environ.get("KEYMAN_DOCKER"): +if SystemEnv.get("KEYMAN_DOCKER"): import sys import redis import logging sys.path.append("/collectoss") - conn = redis.Redis.from_url(os.environ.get("REDIS_CONN_STRING")) + conn = redis.Redis.from_url(SystemEnv.get("REDIS_CONN_STRING")) # Just log to stdout if we're running in docker logger = logging.Logger("KeyOrchestrator") From 9ed52b80e8693228db4b25d10ac2d508d0c4dd92 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 16:38:47 -0400 Subject: [PATCH 17/36] Swap out only usage of deprecated Environment class Signed-off-by: Adrian Edwards --- collectoss/api/view/init.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/collectoss/api/view/init.py b/collectoss/api/view/init.py index ab4708793..1ab68912c 100644 --- a/collectoss/api/view/init.py +++ b/collectoss/api/view/init.py @@ -1,13 +1,11 @@ import os from pathlib import Path -from .server import Environment from collectoss.application.logs import SystemLogger import secrets, yaml - -env = Environment() +from collectoss.application.environment import SystemEnv # load configuration files and initialize globals -configFile = Path(env.setdefault("CONFIG_LOCATION", "config.yml")) +configFile = Path(SystemEnv.get("CONFIG_LOCATION") or "config.yml") settings = {} From a6070fd0f0c6e6ca5bd9fc2d78eb0946b3765490 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 8 May 2026 16:39:13 -0400 Subject: [PATCH 18/36] Remove no-longer-used Environment class in API Signed-off-by: Adrian Edwards --- collectoss/api/view/server/Environment.py | 58 ----------------------- collectoss/api/view/server/__init__.py | 3 +- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 collectoss/api/view/server/Environment.py diff --git a/collectoss/api/view/server/Environment.py b/collectoss/api/view/server/Environment.py deleted file mode 100644 index 4d35bc70a..000000000 --- a/collectoss/api/view/server/Environment.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -from typing_extensions import deprecated - -class Environment: - """ - This class is used to make dealing with environment variables easier. It - allows you to set multiple environment variables at once, and to get items - with subscript notation without needing to deal with the particularities of - non-existent values. - """ - @deprecated("use collectoss.application.environment.SystemEnv instead") - def __init__(self, **kwargs): - for (key, value) in kwargs.items(): - self[key] = value - - @deprecated("use collectoss.application.environment.SystemEnv instead") - def setdefault(self, key, value): - if not self[key]: - self[key] = value - return value - return self[key] - - @deprecated("use collectoss.application.environment.SystemEnv instead") - def setall(self, **kwargs): - result = {} - for (key, value) in kwargs.items(): - if self[key]: - result[key] = self[key] - self[key] = value - - @deprecated("use collectoss.application.environment.SystemEnv instead") - def getany(self, *args): - result = {} - for arg in args: - if self[arg]: - result[arg] = self[arg] - return result - - @deprecated("use collectoss.application.environment.SystemEnv instead") - def as_type(self, type, key): - if self[key]: - return type(self[key]) - return None - - def __getitem__(self, key): - return os.getenv(key) - - def __setitem__(self, key, value): - os.environ[key] = str(value) - - def __len__(self)-> int: - return len(os.environ) - - def __str__(self)-> str: - return str(os.environ) - - def __iter__(self): - return (item for item in os.environ.items()) \ No newline at end of file diff --git a/collectoss/api/view/server/__init__.py b/collectoss/api/view/server/__init__.py index e919a597a..98ce903be 100644 --- a/collectoss/api/view/server/__init__.py +++ b/collectoss/api/view/server/__init__.py @@ -1,2 +1 @@ -from .LoginException import LoginException -from .Environment import Environment \ No newline at end of file +from .LoginException import LoginException \ No newline at end of file From 67bf77726d50d780e2125ec43a915dfe46e1f51c Mon Sep 17 00:00:00 2001 From: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> Date: Fri, 29 May 2026 17:38:44 -0400 Subject: [PATCH 19/36] Refactor extract_prefix Co-authored-by: Shlok Gilda Signed-off-by: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> --- collectoss/application/environment.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 3a28c12a9..9c5b3cf65 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -16,16 +16,13 @@ def extract_prefix(key: str, prefixes: list[str], separator = "_") -> Optional[s Returns: str: The detected prefix (including any separators) if any, otherwise None """ - prefix_len = 0 + k = key.upper() for p in prefixes: - p = p.upper() - k = key.upper() - if k.startswith(p): - prefix_len += len(p) - - if k[prefix_len] == separator: - prefix_len += len(separator) - return key[0:prefix_len] + p_up = p.upper() + if k == p_up: + return key[:len(p)] + if k.startswith(p_up + separator): + return key[:len(p) + len(separator)] return None From ec5873184b72b32dfba78989a1ee45f62a31a988 Mon Sep 17 00:00:00 2001 From: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> Date: Fri, 29 May 2026 17:39:06 -0400 Subject: [PATCH 20/36] refactor get_bool Co-authored-by: Shlok Gilda Signed-off-by: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> --- collectoss/application/environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/collectoss/application/environment.py b/collectoss/application/environment.py index 9c5b3cf65..22a8c95d4 100644 --- a/collectoss/application/environment.py +++ b/collectoss/application/environment.py @@ -67,7 +67,9 @@ def get_bool(cls, key:str, default: bool, prefixes = _prefixes) -> bool: """gets a value from the environment and cast it to a boolean """ raw_val = cls.get(key, None, prefixes) - return raw_val.lower() in ('true', '1', 't', 'y', 'yes') if raw_val else default + if raw_val is None: + return default + return raw_val.lower() in ('true', '1', 't', 'y', 'yes') @classmethod def set(cls, key: str, value: str, overwrite=True) -> None: From b47a836f259c9d0d4736fa35d0d25af70a69497d Mon Sep 17 00:00:00 2001 From: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> Date: Fri, 29 May 2026 17:39:42 -0400 Subject: [PATCH 21/36] fix ~ path expansion in default scorecard value Co-authored-by: Shlok Gilda Signed-off-by: Adrian Edwards <17362949+MoralCode@users.noreply.github.com> --- collectoss/tasks/git/dependency_tasks/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index 21f24246a..3bd2aaab2 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -80,7 +80,7 @@ def generate_scorecard(logger, repo_git): command = '--repo=' + path #this is path where our scorecard project is located - path_to_scorecard = SystemEnv.get('SCORECARD_DIR', (SystemEnv.get('HOME') or "~") + '/scorecard') + path_to_scorecard = SystemEnv.get('SCORECARD_DIR', os.path.expanduser('~/scorecard')) #setting the environmental variable which is required by scorecard key_handler = GithubApiKeyHandler(logger) From 31f25c18d64d0c94bd9bae2fd2448e3422ee87d0 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 20 May 2026 16:14:40 -0400 Subject: [PATCH 22/36] replace a bunch of env var names the application accesses impact: low due to the new env variable interpretation layer Signed-off-by: Adrian Edwards --- collectoss/api/routes/auggie.py | 6 +++--- collectoss/application/cli/__init__.py | 4 ++-- collectoss/application/cli/api.py | 2 +- collectoss/application/cli/backend.py | 8 ++++---- collectoss/application/cli/collection.py | 2 +- collectoss/application/cli/db.py | 4 ++-- collectoss/application/config.py | 2 +- .../tasks/git/util/facade_worker/facade_worker/config.py | 2 +- collectoss/tasks/init/celery_app.py | 2 +- collectoss/tasks/start_tasks.py | 4 ++-- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/collectoss/api/routes/auggie.py b/collectoss/api/routes/auggie.py index 6d036045a..4cde77084 100644 --- a/collectoss/api/routes/auggie.py +++ b/collectoss/api/routes/auggie.py @@ -254,7 +254,7 @@ def get_auggie_user(): # return Response(response=response, status=200, mimetype="application/json") ## From Method profile_name = 'collectoss' - if SystemEnv.get('AUGUR_IS_PROD'): + if SystemEnv.get('COLLECTOSS_IS_PROD'): profile_name = 'default' client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') response = client.get_item( @@ -280,7 +280,7 @@ def update_auggie_user_tracking(): # return Response(response=response, status=200, mimetype="application/json") ## From Method profile_name = 'collectoss' - if SystemEnv.get('AUGUR_IS_PROD'): + if SystemEnv.get('COLLECTOSS_IS_PROD'): profile_name = 'default' client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') response = client.update_item( @@ -342,7 +342,7 @@ def slack_login(): email = user_response["user"]["email"] profile_name = 'collectoss' - if SystemEnv.get('AUGUR_IS_PROD'): + if SystemEnv.get('COLLECTOSS_IS_PROD'): profile_name = 'default' print("Making Boto3 Session") client = boto3.Session(region_name='us-east-1', diff --git a/collectoss/application/cli/__init__.py b/collectoss/application/cli/__init__.py index b398614e2..18fac2f0a 100644 --- a/collectoss/application/cli/__init__.py +++ b/collectoss/application/cli/__init__.py @@ -67,11 +67,11 @@ def new_func(ctx, *args, **kwargs): return ctx.invoke(function_db_connection, *args, **kwargs) except OperationalError as e: - db_environment_var = SystemEnv.get("AUGUR_DB") + db_environment_var = SystemEnv.get("COLLECTOSS_DB") # determine the location to print in error string if db_environment_var: - location = f"the AUGUR_DB environment variable\nAUGUR_DB={SystemEnv.get('AUGUR_DB')}" + location = f"the AUGUR_DB environment variable\nAUGUR_DB={SystemEnv.get('COLLECTOSS_DB')}" else: with open("db.config.json", 'r') as f: db_config = json.load(f) diff --git a/collectoss/application/cli/api.py b/collectoss/application/cli/api.py index 4f7077a78..0c567c590 100644 --- a/collectoss/application/cli/api.py +++ b/collectoss/application/cli/api.py @@ -38,7 +38,7 @@ def start(ctx, development, port): """Start CollectOSS's backend server.""" try: - if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index d4586083b..0bbc5d829 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -62,7 +62,7 @@ def start(ctx, disable_collection, development, pidfile, port): signal.signal(signal.SIGINT, manager.shutdown_signal_handler) try: - if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( @@ -145,7 +145,7 @@ def start(ctx, disable_collection, development, pidfile, port): manager.keypub = keypub if not disable_collection: - if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": orchestrator = subprocess.Popen("python keyman/Orchestrator.py".split()) # Wait for orchestrator startup @@ -356,10 +356,10 @@ def export_env(config): Exports your GitHub key and database credentials """ - export_file = open(SystemEnv.get('AUGUR_EXPORT_FILE') or 'collectoss_export_env.sh', 'w+') + export_file = open(SystemEnv.get('COLLECTOSS_EXPORT_FILE') or 'collectoss_export_env.sh', 'w+') export_file.write('#!/bin/bash') export_file.write('\n') - env_file = open(SystemEnv.get('AUGUR_ENV_FILE') or 'docker_env.txt', 'w+') + env_file = open(SystemEnv.get('COLLECTOSS_ENV_FILE') or 'docker_env.txt', 'w+') for env_var in config.get_env_config().items(): if "LOG" not in env_var[0]: diff --git a/collectoss/application/cli/collection.py b/collectoss/application/cli/collection.py index 5127a8d17..7d7186146 100644 --- a/collectoss/application/cli/collection.py +++ b/collectoss/application/cli/collection.py @@ -46,7 +46,7 @@ def start(ctx, development): """Start CollectOSS's backend server.""" try: - if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": + if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": raise_open_file_limit(100000) except Exception as e: logger.error( diff --git a/collectoss/application/cli/db.py b/collectoss/application/cli/db.py index a93668b3e..38643421f 100644 --- a/collectoss/application/cli/db.py +++ b/collectoss/application/cli/db.py @@ -380,7 +380,7 @@ def get_api_key(ctx): short_help="Check the ~/.pgpass file for CollectOSS's database credentials", ) def check_pgpass(): - db_environment_var = SystemEnv.get("AUGUR_DB") + db_environment_var = SystemEnv.get("COLLECTOSS_DB") if db_environment_var: # gets the user, passowrd, host, port, and database_name out of environment variable # assumes database string of structure //:@:/ @@ -496,7 +496,7 @@ def run_psql_command_in_database(target_type, target): logger.error("Invalid target type. Exiting...") exit(1) - db_environment_var = SystemEnv.get("AUGUR_DB") + db_environment_var = SystemEnv.get("COLLECTOSS_DB") # db_json_file_location = os.getcwd() + "/db.config.json" # db_json_exists = os.path.exists(db_json_file_location) diff --git a/collectoss/application/config.py b/collectoss/application/config.py index 051235323..16f62b5ad 100644 --- a/collectoss/application/config.py +++ b/collectoss/application/config.py @@ -29,7 +29,7 @@ def get_development_flag_from_config(): return flag def get_development_flag(): - return SystemEnv.get("AUGUR_DEV") or get_development_flag_from_config() or False + return SystemEnv.get("COLLECTOSS_DEV") or get_development_flag_from_config() or False def redact_setting_value(section_name, setting_name, value): value_redacted = value if section_name != "Keys" else "REDACTED" diff --git a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py index 9db7d8866..f6d5aa465 100644 --- a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py +++ b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py @@ -46,7 +46,7 @@ def get_database_args_from_env(): - db_str = SystemEnv.get("AUGUR_DB") + db_str = SystemEnv.get("COLLECTOSS_DB") try: db_json_file_location = os.getcwd() + "/db.config.json" except FileNotFoundError: diff --git a/collectoss/tasks/init/celery_app.py b/collectoss/tasks/init/celery_app.py index a33e1e961..22fd34872 100644 --- a/collectoss/tasks/init/celery_app.py +++ b/collectoss/tasks/init/celery_app.py @@ -63,7 +63,7 @@ tasks = start_tasks + github_tasks + gitlab_tasks + git_tasks + materialized_view_tasks + frontend_tasks -if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": +if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": tasks += data_analysis_tasks redis_db_number, redis_conn_string = get_redis_conn_values() diff --git a/collectoss/tasks/start_tasks.py b/collectoss/tasks/start_tasks.py index e7f86b73b..9c6c35c57 100644 --- a/collectoss/tasks/start_tasks.py +++ b/collectoss/tasks/start_tasks.py @@ -15,7 +15,7 @@ from collectoss.tasks.github.repo_info.tasks import * from collectoss.tasks.github.releases.tasks import * from collectoss.application.environment import SystemEnv -if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1": +if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1": from collectoss.tasks.data_analysis import * from collectoss.tasks.github.detect_move.tasks import detect_github_repo_move_core, detect_github_repo_move_secondary from collectoss.tasks.github.releases.tasks import collect_releases @@ -39,7 +39,7 @@ from collectoss.application.db.lib import execute_sql, get_session from collectoss.application.config import SystemConfig -RUNNING_DOCKER = SystemEnv.get('AUGUR_DOCKER_DEPLOY') == "1" +RUNNING_DOCKER = SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') == "1" CELERY_GROUP_TYPE = type(group()) CELERY_CHAIN_TYPE = type(chain()) From a9101b5a1387bf4baf6e6a4d5ef7212b1e8017ca Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 20 May 2026 16:16:06 -0400 Subject: [PATCH 23/36] update env var names in RST docs Signed-off-by: Adrian Edwards --- docs/source/deployment/production.rst | 16 ++++----- .../create-a-metric/api-development.rst | 6 ++-- .../create-a-metric/metrics-steps.rst | 2 +- .../workers/creating_a_new_worker.rst | 2 +- docs/source/docker/docker-compose.rst | 12 +++---- docs/source/docker/getting-started.rst | 24 ++++++------- docs/source/docker/quick-start.rst | 28 +++++++-------- .../command-line-interface/backend.rst | 36 +++++++++---------- .../command-line-interface/configure.rst | 16 ++++----- docs/source/getting-started/using-docker.rst | 16 ++++----- 10 files changed, 79 insertions(+), 79 deletions(-) diff --git a/docs/source/deployment/production.rst b/docs/source/deployment/production.rst index e65a987f1..614737256 100644 --- a/docs/source/deployment/production.rst +++ b/docs/source/deployment/production.rst @@ -11,10 +11,10 @@ Environment Variables CollectOSS uses several environment variables in production. Make sure to configure the ones relevant to your deployment: -- ``AUGUR_RESET_LOGS`` : Controls automatic log reset on server startup -- ``AUGUR_DB`` : PostgreSQL database connection string (used if variable not set) +- ``COLLECTOSS_RESET_LOGS`` : Controls automatic log reset on server startup +- ``COLLECTOSS_DB`` : PostgreSQL database connection string (used if variable not set) -AUGUR_RESET_LOGS +COLLECTOSS_RESET_LOGS ---------------- **Description:** @@ -27,7 +27,7 @@ boolean `True` : CollectOSS clears old logs at startup. **Environment Variable:** -AUGUR_RESET_LOGS +COLLECTOSS_RESET_LOGS **Notes:** If set to `False`, CollectOSS will not reset logs automatically. Administrators must ensure log rotation or cleanup is handled manually. @@ -36,9 +36,9 @@ If set to `False`, CollectOSS will not reset logs automatically. Administrators .. code-block:: bash - export AUGUR_RESET_LOGS=False + export COLLECTOSS_RESET_LOGS=False -AUGUR_DB +COLLECTOSS_DB -------- **Description:** @@ -48,10 +48,10 @@ Specifies the connection string for the PostgreSQL database used by CollectOSS. string **Default:** -Docker container database (if `AUGUR_DB` is not specified) +Docker container database (if `COLLECTOSS_DB` is not specified) **Environment Variable:** -AUGUR_DB +COLLECTOSS_DB Related Resources ----------------- diff --git a/docs/source/development-guide/create-a-metric/api-development.rst b/docs/source/development-guide/create-a-metric/api-development.rst index 8aea48aac..12a010465 100644 --- a/docs/source/development-guide/create-a-metric/api-development.rst +++ b/docs/source/development-guide/create-a-metric/api-development.rst @@ -11,13 +11,13 @@ JSON Metrics are here: .. code-block:: bash - $ AUGUR_HOME/collectoss/metrics + $ COLLECTOSS_HOME/collectoss/metrics Visualization Metrics are here: .. code-block:: bash - $ AUGUR_HOME/collectoss/routes + $ COLLECTOSS_HOME/collectoss/routes Existing metrics files (JSON Metric) "Standard Metrics": @@ -46,7 +46,7 @@ You can see that one of the imports is our standard metric import from the util .. code-block:: python - AUGUR_HOME/collectoss/routes/util.py + COLLECTOSS_HOME/collectoss/routes/util.py All "Standard Metrics" share declaration and a method signature diff --git a/docs/source/development-guide/create-a-metric/metrics-steps.rst b/docs/source/development-guide/create-a-metric/metrics-steps.rst index 5604c422b..a2fb24a02 100644 --- a/docs/source/development-guide/create-a-metric/metrics-steps.rst +++ b/docs/source/development-guide/create-a-metric/metrics-steps.rst @@ -11,7 +11,7 @@ There are many paths, but we usually follow something along these lines: 2. Sometimes, there are metrics endpoints that integrate, or visualize several metrics. 3. Determine what tables in the CollectOSS Schema contain the data we need to develop this metric 4. Construct a very basic query that does the work of joining those tables in a minimal way so we have a "baseline query." -5. Refine the query so that it takes the standard inputs for a "standard metric" if that's what type it is; alternatively, look at non-standard metrics as they are defined in ``AUGUR_HOME/collectoss/routes``, or one of the visualization metrics in ``AUGUR_HOME/collectoss/routes/contributor.py``, ``AUGUR_HOME/collectoss/routes/pull_requests.py`` or ``AUGUR_HOME/collectoss/routes/nonstandard_metrics.py``. (This step is explained in the next section.) +5. Refine the query so that it takes the standard inputs for a "standard metric" if that's what type it is; alternatively, look at non-standard metrics as they are defined in ``COLLECTOSS_HOME/collectoss/routes``, or one of the visualization metrics in ``COLLECTOSS_HOME/collectoss/routes/contributor.py``, ``COLLECTOSS_HOME/collectoss/routes/pull_requests.py`` or ``COLLECTOSS_HOME/collectoss/routes/nonstandard_metrics.py``. (This step is explained in the next section.) Example Query diff --git a/docs/source/development-guide/workers/creating_a_new_worker.rst b/docs/source/development-guide/workers/creating_a_new_worker.rst index 4e713c4ac..a34d73f4b 100644 --- a/docs/source/development-guide/workers/creating_a_new_worker.rst +++ b/docs/source/development-guide/workers/creating_a_new_worker.rst @@ -132,7 +132,7 @@ In the Worker block you need to add something like this: There should NOT be a comma after the final entry in each block. -ALSO, if you wanted to have those blocks installed with auger itself when you do the PR, you need to add them to the `$AUGUR_ROOT/collectoss/config.py` file. The recommended way is to set a port range not already in use and assign a random variable range with the others, like this `your_new_worker_p = randint(56500, 56999)` ... its totally ok to compress a couple other port ranges for this process. +ALSO, if you wanted to have those blocks installed with auger itself when you do the PR, you need to add them to the `$COLLECTOSS_ROOT/collectoss/config.py` file. The recommended way is to set a port range not already in use and assign a random variable range with the others, like this `your_new_worker_p = randint(56500, 56999)` ... its totally ok to compress a couple other port ranges for this process. You can copy the housekeeper block verbatim from what you added to your own `augur.config.json`. For the worker block, in the `config.py` it would look like this: diff --git a/docs/source/docker/docker-compose.rst b/docs/source/docker/docker-compose.rst index ae38fab6c..4afeafd33 100644 --- a/docs/source/docker/docker-compose.rst +++ b/docs/source/docker/docker-compose.rst @@ -27,16 +27,16 @@ This section of the documentation details how to use CollectOSS's Docker Compose .. warning:: Don't forget to provide your external database credentials in a file called ``.env`` file. Make sure all the following environment variables are specified, keep placeholder values if you don't need some of them. - Don't specify AUGUR_DB if you want the docker database to be used. + Don't specify COLLECTOSS_DB if you want the docker database to be used. Example .env: .. code:: - AUGUR_GITHUB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx - AUGUR_GITHUB_USERNAME=usernameGithub - AUGUR_GITLAB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx - AUGUR_GITLAB_USERNAME=usernameGitlab - AUGUR_DB=yourDBString + COLLECTOSS_GITHUB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx + COLLECTOSS_GITHUB_USERNAME=usernameGithub + COLLECTOSS_GITLAB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx + COLLECTOSS_GITLAB_USERNAME=usernameGitlab + COLLECTOSS_DB=yourDBString diff --git a/docs/source/docker/getting-started.rst b/docs/source/docker/getting-started.rst index 0648236a5..db6822b79 100644 --- a/docs/source/docker/getting-started.rst +++ b/docs/source/docker/getting-started.rst @@ -31,14 +31,14 @@ with the following fields (don't remove any variable, keep placeholder values if .. code:: python - AUGUR_DB=collectoss - AUGUR_DB_USER=collectoss - AUGUR_DB_PASSWORD=password_here + COLLECTOSS_DB=collectoss + COLLECTOSS_DB_USER=collectoss + COLLECTOSS_DB_PASSWORD=password_here - AUGUR_GITHUB_API_KEY=ghp_value_here - AUGUR_GITHUB_USERNAME=gh_username - AUGUR_GITLAB_API_KEY=placeholder - AUGUR_GITLAB_USERNAME=placeholder + COLLECTOSS_GITHUB_API_KEY=ghp_value_here + COLLECTOSS_GITHUB_USERNAME=gh_username + COLLECTOSS_GITLAB_API_KEY=placeholder + COLLECTOSS_GITLAB_USERNAME=placeholder Then run: @@ -98,11 +98,11 @@ You can provide your own ``.env`` file to pull from. The file should have the be .. code:: - AUGUR_GITHUB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx - AUGUR_GITHUB_USERNAME=usernameGithub - AUGUR_GITLAB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx - AUGUR_GITLAB_USERNAME=usernameGitlab - AUGUR_DB=yourDBString + COLLECTOSS_GITHUB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx + COLLECTOSS_GITHUB_USERNAME=usernameGithub + COLLECTOSS_GITLAB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxx + COLLECTOSS_GITLAB_USERNAME=usernameGitlab + COLLECTOSS_DB=yourDBString Now that you've created your config file or are ready to generate it yourself, you're ready to `get going `_ . diff --git a/docs/source/docker/quick-start.rst b/docs/source/docker/quick-start.rst index 86b552ea3..c71d9dfa2 100644 --- a/docs/source/docker/quick-start.rst +++ b/docs/source/docker/quick-start.rst @@ -13,14 +13,14 @@ Before you get off to such a quick start, go ahead and .. code:: python - AUGUR_DB=collectoss - AUGUR_DB_USER=collectoss - AUGUR_DB_PASSWORD=password_here + COLLECTOSS_DB=collectoss + COLLECTOSS_DB_USER=collectoss + COLLECTOSS_DB_PASSWORD=password_here - AUGUR_GITHUB_API_KEY=ghp_value_here - AUGUR_GITHUB_USERNAME=gh_username - AUGUR_GITLAB_API_KEY=placeholder - AUGUR_GITLAB_USERNAME=placeholder + COLLECTOSS_GITHUB_API_KEY=ghp_value_here + COLLECTOSS_GITHUB_USERNAME=gh_username + COLLECTOSS_GITLAB_API_KEY=placeholder + COLLECTOSS_GITLAB_USERNAME=placeholder 5. Build the container using one of the following commands: @@ -57,14 +57,14 @@ And collectoss should be up and running! .. code-block:: - AUGUR_DB=collectoss - AUGUR_DB_USER=collectoss - AUGUR_DB_PASSWORD=password_here + COLLECTOSS_DB=collectoss + COLLECTOSS_DB_USER=collectoss + COLLECTOSS_DB_PASSWORD=password_here - AUGUR_GITHUB_API_KEY=ghp_value_here - AUGUR_GITHUB_USERNAME=gh_username - AUGUR_GITLAB_API_KEY=placeholder - AUGUR_GITLAB_USERNAME=placeholder + COLLECTOSS_GITHUB_API_KEY=ghp_value_here + COLLECTOSS_GITHUB_USERNAME=gh_username + COLLECTOSS_GITLAB_API_KEY=placeholder + COLLECTOSS_GITLAB_USERNAME=placeholder 4. Execute the code from the base directory of the CollectOSS repository: diff --git a/docs/source/getting-started/command-line-interface/backend.rst b/docs/source/getting-started/command-line-interface/backend.rst index d53fd36ae..2adcce0ef 100644 --- a/docs/source/getting-started/command-line-interface/backend.rst +++ b/docs/source/getting-started/command-line-interface/backend.rst @@ -145,29 +145,29 @@ Successful output looks like: .. code-block:: bash - > CLI: [util.export_env] [INFO] Exporting AUGUR_GITHUB_API_KEY - > CLI: [util.export_env] [INFO] Exporting AUGUR_DB_HOST - > CLI: [util.export_env] [INFO] Exporting AUGUR_DB_NAME - > CLI: [util.export_env] [INFO] Exporting AUGUR_DB_PORT - > CLI: [util.export_env] [INFO] Exporting AUGUR_DB_USER - > CLI: [util.export_env] [INFO] Exporting AUGUR_DB_PASSWORD + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_GITHUB_API_KEY + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_DB_HOST + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_DB_NAME + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_DB_PORT + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_DB_USER + > CLI: [util.export_env] [INFO] Exporting COLLECTOSS_DB_PASSWORD # contents of collectoss_export_env.sh #!/bin/bash - export AUGUR_GITHUB_API_KEY="your_key_here" - export AUGUR_DB_HOST="your_host" - export AUGUR_DB_NAME="your_db_name" - export AUGUR_DB_PORT="your_db_port" - export AUGUR_DB_USER="your_db_user" - export AUGUR_DB_PASSWORD="your_db_password" + export COLLECTOSS_GITHUB_API_KEY="your_key_here" + export COLLECTOSS_DB_HOST="your_host" + export COLLECTOSS_DB_NAME="your_db_name" + export COLLECTOSS_DB_PORT="your_db_port" + export COLLECTOSS_DB_USER="your_db_user" + export COLLECTOSS_DB_PASSWORD="your_db_password" # contents of docker_env.txt - AUGUR_GITHUB_API_KEY="your_key_here" - AUGUR_DB_HOST="your_host" - AUGUR_DB_NAME="your_db_name" - AUGUR_DB_PORT="your_db_port" - AUGUR_DB_USER="your_db_user" - AUGUR_DB_PASSWORD="your_db_password" + COLLECTOSS_GITHUB_API_KEY="your_key_here" + COLLECTOSS_DB_HOST="your_host" + COLLECTOSS_DB_NAME="your_db_name" + COLLECTOSS_DB_PORT="your_db_port" + COLLECTOSS_DB_USER="your_db_user" + COLLECTOSS_DB_PASSWORD="your_db_password" ``repo-reset`` diff --git a/docs/source/getting-started/command-line-interface/configure.rst b/docs/source/getting-started/command-line-interface/configure.rst index 5659cf6ec..89350bc1a 100644 --- a/docs/source/getting-started/command-line-interface/configure.rst +++ b/docs/source/getting-started/command-line-interface/configure.rst @@ -12,19 +12,19 @@ The ``init`` command is used to create a configuration file, by default named `` Each of the available parameters is optional, and can also be configured using an existing environment variable. Below is the list of available parameters, their defaults, and the corresponding environment variable. ---db_name Database name for your data collection database. Defaults to ``augur``. Set by the ``AUGUR_DB_NAME`` environment variable +--db_name Database name for your data collection database. Defaults to ``augur``. Set by the ``COLLECTOSS_DB_NAME`` environment variable ---db_host Host for your data collection database. Defaults to ``localhost``. Set by the ``AUGUR_DB_HOST`` environment variable +--db_host Host for your data collection database. Defaults to ``localhost``. Set by the ``COLLECTOSS_DB_HOST`` environment variable ---db_user User for your data collection database. Defaults to ``augur``. Set by the ``AUGUR_DB_USER`` environment variable +--db_user User for your data collection database. Defaults to ``augur``. Set by the ``COLLECTOSS_DB_USER`` environment variable ---db_port Port for your data collection database. Defaults to ``5432``. Set by the ``AUGUR_DB_PORT`` environment variable +--db_port Port for your data collection database. Defaults to ``5432``. Set by the ``COLLECTOSS_DB_PORT`` environment variable ---db_password Password for your data collection database. Defaults to ``augur``. Set by the ``AUGUR_DB_PASSWORD`` environment variable +--db_password Password for your data collection database. Defaults to ``augur``. Set by the ``COLLECTOSS_DB_PASSWORD`` environment variable ---github_api_key GitHub API key for data collection from the GitHub API. Defaults to ``key``. Set by the ``AUGUR_GITHUB_API_KEY`` environment variable +--github_api_key GitHub API key for data collection from the GitHub API. Defaults to ``key``. Set by the ``COLLECTOSS_GITHUB_API_KEY`` environment variable ---facade_repo_directory The directory on this machine where Facade should store its cloned repos. Defaults to ``repos/``. Set by the ``AUGUR_FACADE_REPO_DIRECTORY`` environment variable +--facade_repo_directory The directory on this machine where Facade should store its cloned repos. Defaults to ``repos/``. Set by the ``COLLECTOSS_FACADE_REPO_DIRECTORY`` environment variable --rc-config-file Path to an existing CollectOSS config file whose values will be used as the defaults. Defaults to ``None``. This parameter does not support being set by an environment variable. @@ -41,7 +41,7 @@ Example usage\: $ uv run collectoss config init --db_name "db_name" --db_host "host" --db_port "port" --db_user "db_user" --db_password "password" --github_api_key "github_api_key" --facade_repo_directory "facade_repo_directory" # to generate an augur.config.json given all credentials and environment variables - $ uv run collectoss config init --db_name $AUGUR_DB_NAME --db_host $AUGUR_DB_HOST --db_port $AUGUR_DB_PORT --db_user $AUGUR_DB_DB_USER --db_password $AUGUR_DB_PASSWORD --github_api_key $AUGUR_GITHUB_API_KEY --facade_repo_directory $AUGUR_FACADE_REPO_DIRECTORY + $ uv run collectoss config init --db_name $COLLECTOSS_DB_NAME --db_host $COLLECTOSS_DB_HOST --db_port $COLLECTOSS_DB_PORT --db_user $COLLECTOSS_DB_DB_USER --db_password $COLLECTOSS_DB_PASSWORD --github_api_key $COLLECTOSS_GITHUB_API_KEY --facade_repo_directory $COLLECTOSS_FACADE_REPO_DIRECTORY # successful output looks like: > CLI: [config.init] [INFO] Config written to /Users/carter/.collectoss/augur.config.json diff --git a/docs/source/getting-started/using-docker.rst b/docs/source/getting-started/using-docker.rst index 5028d5c3a..c427372b1 100644 --- a/docs/source/getting-started/using-docker.rst +++ b/docs/source/getting-started/using-docker.rst @@ -14,14 +14,14 @@ the following resources (or more). .. code:: python - AUGUR_DB=augur - AUGUR_DB_USER=augur - AUGUR_DB_PASSWORD=password_here - - AUGUR_GITHUB_API_KEY=ghp_value_here - AUGUR_GITHUB_USERNAME=gh_username - AUGUR_GITLAB_API_KEY=placeholder - AUGUR_GITLAB_USERNAME=placeholder + COLLECTOSS_DB=augur + COLLECTOSS_DB_USER=augur + COLLECTOSS_DB_PASSWORD=password_here + + COLLECTOSS_GITHUB_API_KEY=ghp_value_here + COLLECTOSS_GITHUB_USERNAME=gh_username + COLLECTOSS_GITLAB_API_KEY=placeholder + COLLECTOSS_GITLAB_USERNAME=placeholder 3. Build the container using one of the following commands: From dc7e0902cafab37c50f13930258bfa3aa56443f8 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 27 May 2026 11:30:34 -0400 Subject: [PATCH 24/36] update vars in example env Signed-off-by: Adrian Edwards --- environment.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/environment.txt b/environment.txt index 42d00b9c1..3d4c4a721 100644 --- a/environment.txt +++ b/environment.txt @@ -1,12 +1,12 @@ -AUGUR_DB_HOST=collectoss -AUGUR_DB_NAME=collectoss -AUGUR_DB_USER=collectoss -AUGUR_DB_PASSWORD= +COLLECTOSS_DB_HOST=collectoss +COLLECTOSS_DB_NAME=collectoss +COLLECTOSS_DB_USER=collectoss +COLLECTOSS_DB_PASSWORD= -AUGUR_GITHUB_API_KEY= -AUGUR_GITHUB_USERNAME= -AUGUR_GITLAB_API_KEY= -AUGUR_GITLAB_USERNAME= +COLLECTOSS_GITHUB_API_KEY= +COLLECTOSS_GITHUB_USERNAME= +COLLECTOSS_GITLAB_API_KEY= +COLLECTOSS_GITLAB_USERNAME= -AUGUR_RABBITMQ_USERNAME= -AUGUR_RABBITMQ_PASSWORD= +COLLECTOSS_RABBITMQ_USERNAME= +COLLECTOSS_RABBITMQ_PASSWORD= From fb7d113d1b2b74d8053aa57a262abaa99a349f61 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 27 May 2026 11:32:34 -0400 Subject: [PATCH 25/36] update variable names in docker compose Signed-off-by: Adrian Edwards --- docker-compose.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 55e1127be..e1e8ed8da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,11 @@ services: restart: unless-stopped environment: - "POSTGRES_DB=augur" - - "POSTGRES_USER=${AUGUR_DB_USER:-augur}" - - "POSTGRES_PASSWORD=${AUGUR_DB_PASSWORD:-augur}" + - "POSTGRES_USER=${COLLECTOSS_DB_USER:-augur}" + - "POSTGRES_PASSWORD=${COLLECTOSS_DB_PASSWORD:-augur}" - "PGDATA=/var/lib/postgresql/data/pgdata" ports: - - "${AUGUR_DB_PORT:-5432}:5432" + - "${COLLECTOSS_DB_PORT:-5432}:5432" volumes: - augurpostgres:/var/lib/postgresql/data @@ -36,15 +36,15 @@ services: context: . dockerfile: ./docker/rabbitmq/Dockerfile args: - - RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur} - - RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123} - - RABBIT_MQ_DEFAULT_VHOST=${AUGUR_RABBITMQ_VHOST:-collectoss_vhost} + - RABBIT_MQ_DEFAULT_USER=${COLLECTOSS_RABBITMQ_USERNAME:-augur} + - RABBIT_MQ_DEFAULT_PASSWORD=${COLLECTOSS_RABBITMQ_PASSWORD:-password123} + - RABBIT_MQ_DEFAULT_VHOST=${COLLECTOSS_RABBITMQ_VHOST:-collectoss_vhost} core: image: collectoss:latest build: context: . - dockerfile: ./docker/backend/${AUGUR_TARGET:-Dockerfile} + dockerfile: ./docker/backend/${COLLECTOSS_TARGET:-Dockerfile} volumes: - cache:/cache:rw - config:/config:rw @@ -56,16 +56,16 @@ services: #extra_hosts: # - "host.docker.internal:host-gateway" #Be able to ping services on the local machine environment: - - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@database:5432/augur" - - "AUGUR_DB_SCHEMA_BUILD=1" - - AUGUR_FACADE_REPO_DIRECTORY=/facade - - "AUGUR_FLAGS=$AUGUR_FLAGS" - - "AUGUR_GITHUB_API_KEY=${AUGUR_GITHUB_API_KEY}" - - "AUGUR_GITLAB_API_KEY=${AUGUR_GITLAB_API_KEY}" - - "AUGUR_GITHUB_USERNAME=${AUGUR_GITHUB_USERNAME}" - - "AUGUR_GITLAB_USERNAME=${AUGUR_GITLAB_USERNAME}" + - "COLLECTOSS_DB=postgresql+psycopg2://${COLLECTOSS_DB_USER:-augur}:${COLLECTOSS_DB_PASSWORD:-augur}@database:5432/augur" + - "COLLECTOSS_DB_SCHEMA_BUILD=1" + - COLLECTOSS_FACADE_REPO_DIRECTORY=/facade + - "COLLECTOSS_FLAGS=$COLLECTOSS_FLAGS" + - "COLLECTOSS_GITHUB_API_KEY=${COLLECTOSS_GITHUB_API_KEY}" + - "COLLECTOSS_GITLAB_API_KEY=${COLLECTOSS_GITLAB_API_KEY}" + - "COLLECTOSS_GITHUB_USERNAME=${COLLECTOSS_GITHUB_USERNAME}" + - "COLLECTOSS_GITLAB_USERNAME=${COLLECTOSS_GITLAB_USERNAME}" - REDIS_CONN_STRING=redis://redis:6379 - - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-collectoss_vhost} + - RABBITMQ_CONN_STRING=amqp://${COLLECTOSS_RABBITMQ_USERNAME:-augur}:${COLLECTOSS_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${COLLECTOSS_RABBITMQ_VHOST:-collectoss_vhost} - CONFIG_LOCATION=/config/config.yml - CONFIG_DATADIR=/config - CACHE_DATADIR=/cache @@ -92,9 +92,9 @@ services: # ports: # - 5555:5555 # environment: - # - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@database:5432/augur" + # - "COLLECTOSS_DB=postgresql+psycopg2://${COLLECTOSS_DB_USER:-augur}:${COLLECTOSS_DB_PASSWORD:-augur}@database:5432/augur" # - REDIS_CONN_STRING=redis://redis:6379 - # - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-collectoss_vhost} + # - RABBITMQ_CONN_STRING=amqp://${COLLECTOSS_RABBITMQ_USERNAME:-augur}:${COLLECTOSS_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${COLLECTOSS_RABBITMQ_VHOST:-collectoss_vhost} # depends_on: # - core # - database From bf8b20c3ee9f8e26a686a796cbb7db061a9eb132 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 12:55:57 -0400 Subject: [PATCH 26/36] hard change env var prefix for CLI commands Signed-off-by: Adrian Edwards --- collectoss/application/cli/_multicommand.py | 2 +- collectoss/application/cli/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collectoss/application/cli/_multicommand.py b/collectoss/application/cli/_multicommand.py index 13186e7bb..06aae01de 100644 --- a/collectoss/application/cli/_multicommand.py +++ b/collectoss/application/cli/_multicommand.py @@ -11,7 +11,7 @@ from pathlib import Path # import collectoss.application -CONTEXT_SETTINGS = dict(auto_envvar_prefix='AUGUR') +CONTEXT_SETTINGS = dict(auto_envvar_prefix='COLLECTOSS') class CLIMultiCommand(click.MultiCommand): def __commands_folder(self): diff --git a/collectoss/application/cli/config.py b/collectoss/application/cli/config.py index 2a9a09320..9753f5299 100644 --- a/collectoss/application/cli/config.py +++ b/collectoss/application/cli/config.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -ENVVAR_PREFIX = "AUGUR_" +ENVVAR_PREFIX = "COLLECTOSS_" @click.group('config', short_help='Generate an augur.config.json') @click.pass_context From 43c576afbbb2eec44dc96329f1b2177188e26a65 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 12:56:38 -0400 Subject: [PATCH 27/36] attempt to add transitional variables for the specific existing places where env vars are explicitly needed in the CLI Signed-off-by: Adrian Edwards --- collectoss/application/cli/config.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/collectoss/application/cli/config.py b/collectoss/application/cli/config.py index 9753f5299..50641439e 100644 --- a/collectoss/application/cli/config.py +++ b/collectoss/application/cli/config.py @@ -18,18 +18,21 @@ ENVVAR_PREFIX = "COLLECTOSS_" +def get_transitional_envs(name: str) -> list: + return [ENVVAR_PREFIX + name, "AUGUR_" + name] + @click.group('config', short_help='Generate an augur.config.json') @click.pass_context def cli(ctx): ctx.obj = DatabaseContext() @cli.command('init') -@click.option('--github-api-key', help="GitHub API key for data collection from the GitHub API", envvar=ENVVAR_PREFIX + 'GITHUB_API_KEY') -@click.option('--facade-repo-directory', help="Directory on the database server where Facade should clone repos", envvar=ENVVAR_PREFIX + 'FACADE_REPO_DIRECTORY') -@click.option('--gitlab-api-key', help="GitLab API key for data collection from the GitLab API", envvar=ENVVAR_PREFIX + 'GITLAB_API_KEY') -@click.option('--redis-conn-string', help="String to connect to redis cache", envvar=ENVVAR_PREFIX + 'REDIS_CONN_STRING') -@click.option('--rabbitmq-conn-string', help="String to connect to rabbitmq broker", envvar=ENVVAR_PREFIX + 'RABBITMQ_CONN_STRING') -@click.option('--logs-directory', help="Directory to store logs", envvar=ENVVAR_PREFIX + 'LOGS_DIRECTORY') +@click.option('--github-api-key', help="GitHub API key for data collection from the GitHub API", envvar=get_transitional_envs('GITHUB_API_KEY')) +@click.option('--facade-repo-directory', help="Directory on the database server where Facade should clone repos", envvar=get_transitional_envs('FACADE_REPO_DIRECTORY')) +@click.option('--gitlab-api-key', help="GitLab API key for data collection from the GitLab API", envvar=get_transitional_envs('GITLAB_API_KEY')) +@click.option('--redis-conn-string', help="String to connect to redis cache", envvar=get_transitional_envs('REDIS_CONN_STRING')) +@click.option('--rabbitmq-conn-string', help="String to connect to rabbitmq broker", envvar=get_transitional_envs('RABBITMQ_CONN_STRING')) +@click.option('--logs-directory', help="Directory to store logs", envvar=get_transitional_envs('LOGS_DIRECTORY')) @test_connection @test_db_connection @with_database From d3821659fc47092553e4e1475d0a1024dae89750 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:12:30 -0400 Subject: [PATCH 28/36] factor tests into a class Signed-off-by: Adrian Edwards --- .../test_config/test_environment.py | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_application/test_config/test_environment.py index 6b62f2ec9..587910da7 100644 --- a/tests/test_application/test_config/test_environment.py +++ b/tests/test_application/test_config/test_environment.py @@ -6,75 +6,77 @@ prefixes = ["COLLECTOSS", "OTHER"] -def test_env_extract_prefix(): - assert extract_prefix("OTHER_DB", prefixes) == "OTHER_" - assert extract_prefix("COLLECTOSS_DB", prefixes) == "COLLECTOSS_" +class TestSystemEnv: -def test_env_extract_prefix_default(): - assert extract_prefix("SOME_DB", prefixes) is None - assert extract_prefix("THINGY_DB", prefixes) is None + def test_env_extract_prefix(self): + assert extract_prefix("OTHER_DB", prefixes) == "OTHER_" + assert extract_prefix("COLLECTOSS_DB", prefixes) == "COLLECTOSS_" + def test_env_extract_prefix_default(self): + assert extract_prefix("SOME_DB", prefixes) is None + assert extract_prefix("THINGY_DB", prefixes) is None -def test_env_extract_prefix_unprefixed(): - assert extract_prefix("DB", prefixes) is None -def test_fetching_env(): - # plain - os.environ["COLLECTOSS_NAME"] = "A" - assert SystemEnv.get("COLLECTOSS_NAME") == "A" + def test_env_extract_prefix_unprefixed(self): + assert extract_prefix("DB", prefixes) is None - # fallback handling - os.environ["OTHER_THING"] = "B" - assert SystemEnv.get("COLLECTOSS_THING", None, prefixes) == "B" + def test_fetching_env(self): + # plain + os.environ["COLLECTOSS_NAME"] = "A" + assert SystemEnv.get("COLLECTOSS_NAME") == "A" - # cleanup - del os.environ["COLLECTOSS_NAME"] - del os.environ["OTHER_THING"] + # fallback handling + os.environ["OTHER_THING"] = "B" + assert SystemEnv.get("COLLECTOSS_THING", None, prefixes) == "B" -def test_fetching_env_backwards(): - os.environ["COLLECTOSS_NAME"] = "A" - assert SystemEnv.get("OTHER_NAME", None, prefixes) == "A" + # cleanup + del os.environ["COLLECTOSS_NAME"] + del os.environ["OTHER_THING"] - # cleanup - del os.environ["COLLECTOSS_NAME"] + def test_fetching_env_backwards(self): + os.environ["COLLECTOSS_NAME"] = "A" + assert SystemEnv.get("OTHER_NAME", None, prefixes) == "A" -def test_fetching_env_no_value(): - assert SystemEnv.get("COLLECTOSS_MISSING", None, prefixes) is None + # cleanup + del os.environ["COLLECTOSS_NAME"] -def test_fetching_env_default(): - assert SystemEnv.get("COLLECTOSS_DEFAULT", "SOME", prefixes) == "SOME" + def test_fetching_env_no_value(self): + assert SystemEnv.get("COLLECTOSS_MISSING", None, prefixes) is None -def test_no_known_prefix(): - # fallback handling - os.environ["THING"] = "C" - assert SystemEnv.get("THING", None, prefixes) == "C" + def test_fetching_env_default(self): + assert SystemEnv.get("COLLECTOSS_DEFAULT", "SOME", prefixes) == "SOME" + def test_no_known_prefix(self): + # fallback handling + os.environ["THING"] = "C" + assert SystemEnv.get("THING", None, prefixes) == "C" -def test_get_bool_trues(): - cases = ["1", "true", "True", "TRUE", "y", "Y", "yes", "Yes"] + def test_get_bool_trues(self): - for case in cases: - os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == True - del os.environ["OTHER_BOOL"] + cases = ["1", "true", "True", "TRUE", "y", "Y", "yes", "Yes"] -def test_get_bool_falses(): + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == True + del os.environ["OTHER_BOOL"] - cases = ["0", "false", "False", "FALSE", "n", "N", "no", "No"] + def test_get_bool_falses(self): - for case in cases: - os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", True, prefixes) == False - del os.environ["OTHER_BOOL"] + cases = ["0", "false", "False", "FALSE", "n", "N", "no", "No"] -def test_get_bool_default(): + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", True, prefixes) == False + del os.environ["OTHER_BOOL"] - cases = ["?", "maybe", "Stuff", "333"] + def test_get_bool_default(self): - for case in cases: - os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == False - del os.environ["OTHER_BOOL"] + cases = ["?", "maybe", "Stuff", "333"] - + for case in cases: + os.environ["OTHER_BOOL"] = case + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == False + del os.environ["OTHER_BOOL"] + + From a80349dbc00dd403d923eb3ff4dc4234f7e7af19 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:12:42 -0400 Subject: [PATCH 29/36] import SystemEnv into celery_app Signed-off-by: Adrian Edwards --- collectoss/tasks/init/celery_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/collectoss/tasks/init/celery_app.py b/collectoss/tasks/init/celery_app.py index 22fd34872..4b10af18a 100644 --- a/collectoss/tasks/init/celery_app.py +++ b/collectoss/tasks/init/celery_app.py @@ -17,6 +17,7 @@ from collectoss.application.db import get_engine from collectoss.application.db.lib import get_session from collectoss.application.config import SystemConfig +from collectoss.application.environment import SystemEnv from collectoss.tasks.init import get_redis_conn_values, get_rabbitmq_conn_string from collectoss.application.db.models import Repo from collectoss.tasks.util.collection_state import CollectionState From aeb05b51f4714d51224d800fda7618cd7a350b2b Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:17:51 -0400 Subject: [PATCH 30/36] move test next to the actual known good config tests Signed-off-by: Adrian Edwards --- .../test_config => test_classes}/test_environment.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_application/test_config => test_classes}/test_environment.py (100%) diff --git a/tests/test_application/test_config/test_environment.py b/tests/test_classes/test_environment.py similarity index 100% rename from tests/test_application/test_config/test_environment.py rename to tests/test_classes/test_environment.py From b340f65e07af132cc487fa626a05e9c18be69a0e Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:26:42 -0400 Subject: [PATCH 31/36] add more detailed failure reasons to get_bool tests Signed-off-by: Adrian Edwards --- tests/test_classes/test_environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_classes/test_environment.py b/tests/test_classes/test_environment.py index 587910da7..38c0a99e2 100644 --- a/tests/test_classes/test_environment.py +++ b/tests/test_classes/test_environment.py @@ -58,7 +58,7 @@ def test_get_bool_trues(self): for case in cases: os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == True + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == True, f"value '{case}' should resolve to True" del os.environ["OTHER_BOOL"] def test_get_bool_falses(self): @@ -67,7 +67,7 @@ def test_get_bool_falses(self): for case in cases: os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", True, prefixes) == False + assert SystemEnv.get_bool("OTHER_BOOL", True, prefixes) == False, f"value '{case}' should resolve to False" del os.environ["OTHER_BOOL"] def test_get_bool_default(self): @@ -76,7 +76,7 @@ def test_get_bool_default(self): for case in cases: os.environ["OTHER_BOOL"] = case - assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == False + assert SystemEnv.get_bool("OTHER_BOOL", False, prefixes) == False, f"value '{case}' should resolve to Default value" del os.environ["OTHER_BOOL"] From 1f89a328d8e5af9db5cba7c3f237f5c0d68b827f Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:29:24 -0400 Subject: [PATCH 32/36] split environment tests into two classes Signed-off-by: Adrian Edwards --- tests/test_classes/test_environment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_classes/test_environment.py b/tests/test_classes/test_environment.py index 38c0a99e2..e6621062a 100644 --- a/tests/test_classes/test_environment.py +++ b/tests/test_classes/test_environment.py @@ -6,8 +6,7 @@ prefixes = ["COLLECTOSS", "OTHER"] -class TestSystemEnv: - +class TestExtractPrefix: def test_env_extract_prefix(self): assert extract_prefix("OTHER_DB", prefixes) == "OTHER_" assert extract_prefix("COLLECTOSS_DB", prefixes) == "COLLECTOSS_" @@ -20,6 +19,8 @@ def test_env_extract_prefix_default(self): def test_env_extract_prefix_unprefixed(self): assert extract_prefix("DB", prefixes) is None +class TestSystemEnv: + def test_fetching_env(self): # plain os.environ["COLLECTOSS_NAME"] = "A" From 741fba01b04f9a5c0bbf40538038a0186654060d Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:46:37 -0400 Subject: [PATCH 33/36] apply homedir resolution fix to SCC path as well Signed-off-by: Adrian Edwards --- collectoss/tasks/git/scc_value_tasks/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectoss/tasks/git/scc_value_tasks/core.py b/collectoss/tasks/git/scc_value_tasks/core.py index a526af990..770165522 100644 --- a/collectoss/tasks/git/scc_value_tasks/core.py +++ b/collectoss/tasks/git/scc_value_tasks/core.py @@ -21,7 +21,7 @@ def value_model(logger,repo_git): logger.info(f"Repo ID: {repo_id}, Path: {path}") logger.info('Running scc...') - path_to_scc = SystemEnv.get('SCC_DIR', (SystemEnv.get('HOME') or "~") + '/scc') + path_to_scc = SystemEnv.get('SCC_DIR', os.path.expanduser('~/scc')) required_output = parse_json_from_subprocess_call(logger,['./scc', '-f','json','--by-file', path], cwd=path_to_scc) From de2fc9f702e94cee48e2726a40a4e27728ca1a05 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:51:58 -0400 Subject: [PATCH 34/36] fix docs underline lengths Signed-off-by: Adrian Edwards --- docs/source/deployment/production.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/deployment/production.rst b/docs/source/deployment/production.rst index 614737256..186a38c4c 100644 --- a/docs/source/deployment/production.rst +++ b/docs/source/deployment/production.rst @@ -15,7 +15,7 @@ to your deployment: - ``COLLECTOSS_DB`` : PostgreSQL database connection string (used if variable not set) COLLECTOSS_RESET_LOGS ----------------- +--------------------- **Description:** Controls whether CollectOSS resets its log files every time the server starts. Useful for managing log size or integrating with external log rotation systems. @@ -39,7 +39,7 @@ If set to `False`, CollectOSS will not reset logs automatically. Administrators export COLLECTOSS_RESET_LOGS=False COLLECTOSS_DB --------- +------------- **Description:** Specifies the connection string for the PostgreSQL database used by CollectOSS. If omitted, the default Docker database is used. From 6330627d405fe5fb46850012ebd4da6a4bba57e4 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 17:59:04 -0400 Subject: [PATCH 35/36] remove unused imports Signed-off-by: Adrian Edwards --- collectoss/api/view/init.py | 1 - collectoss/tasks/git/dependency_tasks/core.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/collectoss/api/view/init.py b/collectoss/api/view/init.py index 1ab68912c..b26752af9 100644 --- a/collectoss/api/view/init.py +++ b/collectoss/api/view/init.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from collectoss.application.logs import SystemLogger import secrets, yaml diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index 3bd2aaab2..0648231b0 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -1,7 +1,7 @@ from datetime import datetime import os from collectoss.application.db.models import * -from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value, get_session +from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value from collectoss.application.environment import SystemEnv from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler from collectoss.tasks.git.dependency_tasks.dependency_util import dependency_calculator as dep_calc From a023827a0b2a87acc57dc28cb8673e718a92a267 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 29 May 2026 18:29:44 -0400 Subject: [PATCH 36/36] use SystemEnv for fetching database variable Signed-off-by: Adrian Edwards --- collectoss/application/db/engine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/collectoss/application/db/engine.py b/collectoss/application/db/engine.py index ef582dbed..f0d405cf3 100644 --- a/collectoss/application/db/engine.py +++ b/collectoss/application/db/engine.py @@ -7,6 +7,7 @@ from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine +from collectoss.application.environment import SystemEnv from collectoss.application.db.util import catch_operational_error @@ -61,7 +62,7 @@ def get_database_string() -> str: postgres database string """ - db_environment_var = os.getenv("AUGUR_DB") + db_environment_var = SystemEnv.get("COLLECTOSS_DB") try: current_dir = os.getcwd()