[pull] dev from DedSecInside:dev

.flake8

@@ -1,2 +1,3 @@
 [flake8]
 max-line-length = 119
+exclude = .venv,build,dist

README.md

@@ -39,44 +39,49 @@
 8. Built-in Updater
 9. Build a visual tree of link relationships that can be quickly viewed or saved to a file
 
+### Recent improvements
+- The CLI now allows `--version` and `--update` to run without requiring a URL.
+- The crawler now resolves relative links against the current page and skips unsupported or malformed hrefs more gracefully.
+
 ...(will be updated)
 
 ### Dependencies
 - Tor (Optional)
-- Python ^3.9
-- Poetry (Optional)
-
-### Python Dependencies
+- Python 3.9+
+- pip
 
-(see pyproject.toml or requirements.txt for more details)
+## Quick start
 
-## Installation
-
-### TorBot
+### Local setup
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -r requirements.txt
+python -m pip install -e .
+```
 
-#### Using `venv`
-* If using Python ^3.4,
+### Run the CLI
 ```sh
-python -m venv torbot_venv
-source torbot_venv/bin/activate
-pip install -r requirements.txt
-pip install -e .
-./main.py --help
+python main.py --help
+python main.py --version
+python main.py -u https://example.com --depth 2 --visualize table
 ```
 
-#### Using `docker`
+### Common examples
 ```sh
-docker build -t {image_name} .
+# Gather basic site information
+python main.py -u https://example.com --info
 
-# Running without Tor
-docker run {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+# Save the crawl tree as JSON
+python main.py -u https://example.com --depth 2 --save json
 
-# Running with Tor
-docker run --network="host" {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+# Disable SOCKS5 if you are not using Tor locally
+python main.py -u https://example.com --disable-socks5 --visualize tree
 ```
 
 ### Options
-<pre>
+```text
 usage: Gather and analyze data from Tor sites.
 
 optional arguments:
@@ -92,11 +97,14 @@ optional arguments:
   --save FORMAT         Save results in a file. (tree, JSON)
   --visualize FORMAT    Visualizes tree of data gathered. (tree, JSON, table)
   -i, --info            Info displays basic info of the scanned site
-  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy</pre>
+  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy
+```
+
+> `-u/--url` is required for crawl-related commands such as `--info`, `--save`, and `--visualize`.
 
-* NOTE: -u is a mandatory for crawling
+If you are using Tor locally, keep the SOCKS5 proxy enabled. If you are not using Tor, add `--disable-socks5` to avoid proxy connection errors.
 
-Read more about torrc here : [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md)
+Read more about torrc here: [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md)
 
 ## Curated Features
 - [x] Visualization Module Revamp

docs/CHANGELOG.md

@@ -2,6 +2,12 @@
 --------------------
 All notable changes to this project will be documented in this file.
 
+## 4.1.1
+
+### Fixed
+- Adjusted the CLI so `--version` and `--update` can run without requiring a URL.
+- Improved crawler link parsing so relative links are resolved against the current page while unsupported hrefs are skipped cleanly.
+
 ## 2.1.0
 
 ### Added

main.py

@@ -61,11 +61,6 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None:
     logging_lvl = logging.DEBUG if args.v else logging.INFO
     logging.basicConfig(level=logging_lvl, format=logging_fmt, datefmt=date_fmt)
 
-    # URL is a required argument
-    if not args.url:
-        arg_parser.print_help()
-        sys.exit()
-
     # Print verison then exit
     if args.version:
         print(f"TorBot Version: {version}")
@@ -76,6 +71,11 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None:
         check_version()
         sys.exit()
 
+    # URL is required for crawl-related actions
+    if not args.url:
+        arg_parser.print_help()
+        sys.exit()
+
     socks5_host = args.host
     socks5_port = str(args.port)
     socks5_proxy = f"socks5://{socks5_host}:{socks5_port}"
@@ -123,7 +123,11 @@ def set_arguments() -> argparse.ArgumentParser:
         prog="TorBot", usage="Gather and analayze data from Tor sites."
     )
     parser.add_argument(
-        "-u", "--url", type=str, required=True, help="Specifiy a website link to crawl"
+        "-u",
+        "--url",
+        type=str,
+        default=None,
+        help="Specifiy a website link to crawl",
     )
     parser.add_argument(
         "--depth", type=int, help="Specifiy max depth of crawler (default 1)", default=1

pyproject.toml

@@ -4,51 +4,57 @@ build-backend = "hatchling.build"
 
 [project]
 name = "torbot"
-version = "4.1.0"
+version = "4.1.1"
 authors = [
     { name="Akeem King", email="akeemtlking@gmail.com" },
     { name="PS Narayanan", email="thepsnarayanan@gmail.com" },
 ]
 description = "TorBot is an OSINT tool for the dark web."
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 classifiers = [
   "Programming Language :: Python :: 3",
-  "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: OS Independent",
 ]
 dependencies = [
-  "altgraph==0.17.2",
-  "beautifulsoup4==4.11.1",
-  "certifi==2024.7.4",
-  "charset-normalizer==2.0.12",
-  "decorator==5.1.1",
-  "idna==3.7",
-  "igraph==0.10.6",
-  "joblib==1.2.0",
-  "macholib==1.16",
-  "progress==1.6",
-  "pyinstaller==6.0.0",
-  "pyinstaller-hooks-contrib==2022.7",
-  "PySocks==1.7.1",
-  "python-dotenv==0.20.0",
-  "scikit-learn==1.3.0",
-  "scipy==1.10.0",
-  "six==1.16.0",
-  "sklearn==0.0",
-  "soupsieve==2.3.2.post1",
-  "termcolor==1.1.0",
-  "texttable==1.6.4",
-  "threadpoolctl==3.1.0",
-  "urllib3==2.5.0",
-  "validators==0.20.0",
-  "treelib==1.6.1",
-  "numpy==1.24.4",
-  "unipath==1.1",
-  "httpx[socks]==0.25.0",
-  "tabulate==0.9.0",
-  "phonenumbers==8.13.22",
-  "pytest==7.4.2",
-  "yattag==1.15.1",
-  "toml==0.10.2",
+  "altgraph>=0.17.4",
+  "beautifulsoup4>=4.12.3",
+  "certifi>=2024.7.4",
+  "charset-normalizer>=3.3.2",
+  "decorator>=5.1.1",
+  "idna>=3.15",
+  "igraph>=0.11.4",
+  "joblib>=1.4.2",
+  "macholib>=1.16.3",
+  "progress>=1.6",
+  "pyinstaller>=6.8.0",
+  "pyinstaller-hooks-contrib>=2024.6",
+  "PySocks>=1.7.1",
+  "python-dotenv>=1.2.2",
+  "scikit-learn>=1.5.0",
+  "scipy>=1.13.0",
+  "six>=1.16.0",
+  "sklearn>=0.0",
+  "soupsieve>=2.8.4",
+  "termcolor>=2.4.0",
+  "texttable>=1.7.0",
+  "threadpoolctl>=3.5.0",
+  "urllib3>=2.7.0",
+  "validators>=0.22.0",
+  "treelib>=1.7.1",
+  "numpy>=1.26.4",
+  "unipath>=1.1",
+  "httpx[socks]>=0.27.0",
+  "tabulate>=0.9.0",
+  "phonenumbers>=8.13.37",
+  "pytest>=9.0.3",
+  "yattag>=1.15.2",
+  "toml>=0.10.2",
 ]
+
+[project.scripts]
+torbot = "torbot.cli:main"

requirements.txt

@@ -8,7 +8,7 @@ decorator==5.1.1
 h11==0.14.0
 httpcore==1.0.5
 httpx[socks]==0.27.0
-idna==3.7
+idna==3.15
 igraph==0.11.4
 iniconfig==2.0.0
 joblib==1.4.2
@@ -22,17 +22,17 @@ progress==1.6
 pyinstaller-hooks-contrib==2024.6
 pyinstaller==6.8.0
 pysocks==1.7.1
-pytest==8.1.1
-python-dotenv==1.0.1
+pytest==9.0.3
+python-dotenv==1.2.2
 pywin32-ctypes==0.2.2 ; sys_platform == "win32"
-scikit-learn==1.4.2
+scikit-learn==1.5.0
 scipy==1.13.0
 setuptools==78.1.1
 six==1.16.0
 sklearn==0.0
 sniffio==1.3.1
 socksio==1.0.0
-soupsieve==2.5
+soupsieve==2.8.4
 tabulate==0.9.0
 termcolor==2.4.0
 texttable==1.7.0
@@ -41,6 +41,6 @@ toml==0.10.2
 tomli==2.0.1
 treelib==1.7.1
 unipath==1.1
-urllib3==2.5.0
+urllib3==2.7.0
 validators==0.22.0
 yattag==1.15.2

scripts/publish.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m build
+python3 -m twine check dist/*
+echo "Publishing is ready. Run the following when you have your PyPI token:"
+echo "python3 -m twine upload dist/*"

src/torbot/cli.py

@@ -0,0 +1,28 @@
+import os
+import sys
+
+import toml
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+from main import run as run_cli, set_arguments  # noqa: E402
+
+
+def main() -> None:
+    """Run the TorBot CLI from an installed entry point."""
+    arg_parser = set_arguments()
+    config_file_path = os.path.join(REPO_ROOT, "pyproject.toml")
+    try:
+        with open(config_file_path, "r", encoding="utf-8") as handle:
+            data = toml.load(handle)
+            version = data["project"]["version"]
+    except Exception as exc:
+        raise RuntimeError("unable to find version from pyproject.toml.") from exc
+
+    run_cli(arg_parser, version)
+
+
+if __name__ == "__main__":
+    main()