Add Oracle → PostgreSQL DWH migration tool with daily GitHub Actions schedule

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+    branches: ["**"]
+
+jobs:
+  test:
+    name: Lint & Test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run tests
+        run: pytest tests/ -v --tb=short

.github/workflows/daily_migration.yml

@@ -0,0 +1,86 @@
+name: Daily DWH Migration (Oracle → PostgreSQL)
+
+on:
+  # Run every day at 02:00 UTC
+  schedule:
+    - cron: "0 2 * * *"
+
+  # Allow manual runs from the GitHub UI / API
+  workflow_dispatch:
+    inputs:
+      tables:
+        description: "Comma-separated list of tables to migrate (leave empty for all)"
+        required: false
+        default: ""
+      mode:
+        description: "Migration mode: full or incremental"
+        required: false
+        default: "full"
+      dry_run:
+        description: "Dry run (no data written)"
+        required: false
+        default: "false"
+
+env:
+  # Oracle connection — stored as repository / environment secrets
+  ORACLE_HOST: ${{ secrets.ORACLE_HOST }}
+  ORACLE_PORT: ${{ secrets.ORACLE_PORT }}
+  ORACLE_SERVICE: ${{ secrets.ORACLE_SERVICE }}
+  ORACLE_USER: ${{ secrets.ORACLE_USER }}
+  ORACLE_PASSWORD: ${{ secrets.ORACLE_PASSWORD }}
+
+  # PostgreSQL connection — stored as repository / environment secrets
+  PG_HOST: ${{ secrets.PG_HOST }}
+  PG_PORT: ${{ secrets.PG_PORT }}
+  PG_DATABASE: ${{ secrets.PG_DATABASE }}
+  PG_USER: ${{ secrets.PG_USER }}
+  PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
+
+  # Migration settings — can be overridden at runtime
+  MIGRATION_MODE: ${{ github.event.inputs.mode || 'full' }}
+  MIGRATION_TABLES: ${{ github.event.inputs.tables || '' }}
+  LOG_LEVEL: INFO
+
+jobs:
+  migrate:
+    name: Migrate Oracle DWH → PostgreSQL
+    runs-on: ubuntu-latest
+    environment: production   # use a protected environment for secrets approval
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run migration
+        run: |
+          EXTRA_FLAGS=""
+          if [ "${{ github.event.inputs.dry_run }}" = "true" ]; then
+            EXTRA_FLAGS="--dry-run"
+          fi
+          if [ -n "${{ github.event.inputs.tables }}" ]; then
+            EXTRA_FLAGS="$EXTRA_FLAGS --tables ${{ github.event.inputs.tables }}"
+          fi
+          if [ -n "${{ github.event.inputs.mode }}" ]; then
+            EXTRA_FLAGS="$EXTRA_FLAGS --mode ${{ github.event.inputs.mode }}"
+          fi
+          python migrate.py $EXTRA_FLAGS
+
+      - name: Upload migration log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: migration-log-${{ github.run_id }}
+          path: "*.log"
+          if-no-files-found: ignore
+          retention-days: 30

.gitignore

@@ -0,0 +1,19 @@
+# Python bytecode
+__pycache__/
+*.py[cod]
+*.pyo
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Pytest
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/

README.md

@@ -1 +1,105 @@
-# test
+# Oracle → PostgreSQL DWH Migration Tool
+
+A Python script that migrates tables from an Oracle Data Warehouse to PostgreSQL,
+with a daily GitHub Actions schedule.
+
+## Project structure
+
+```
+├── migrate.py           – Main migration script
+├── type_mapping.py      – Oracle-to-PostgreSQL type conversion
+├── config.py            – Configuration via environment variables
+├── requirements.txt     – Python dependencies
+├── tests/
+│   ├── test_type_mapping.py
+│   ├── test_config.py
+│   └── test_migrate.py
+└── .github/
+    └── workflows/
+        ├── daily_migration.yml  – Scheduled migration (daily @ 02:00 UTC)
+        └── ci.yml               – CI: run tests on every push / PR
+```
+
+## Quick start
+
+### 1. Install dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. Set environment variables
+
+| Variable | Description | Default |
+|---|---|---|
+| `ORACLE_HOST` | Oracle server hostname | **required** |
+| `ORACLE_PORT` | Oracle listener port | `1521` |
+| `ORACLE_SERVICE` | Oracle service name or SID | **required** |
+| `ORACLE_USER` | Oracle schema/user | **required** |
+| `ORACLE_PASSWORD` | Oracle password | **required** |
+| `PG_HOST` | PostgreSQL hostname | **required** |
+| `PG_PORT` | PostgreSQL port | `5432` |
+| `PG_DATABASE` | PostgreSQL database name | **required** |
+| `PG_SCHEMA` | Target PostgreSQL schema | `public` |
+| `PG_USER` | PostgreSQL user | **required** |
+| `PG_PASSWORD` | PostgreSQL password | **required** |
+| `MIGRATION_TABLES` | Comma-separated table list (empty = all) | *(all tables)* |
+| `MIGRATION_BATCH_SIZE` | Rows fetched per batch | `10000` |
+| `MIGRATION_MODE` | `full` (truncate+reload) or `incremental` | `full` |
+| `LOG_LEVEL` | Python logging level | `INFO` |
+
+### 3. Run the migration
+
+```bash
+# Migrate all tables (full reload)
+python migrate.py
+
+# Migrate specific tables
+python migrate.py --tables ORDERS,CUSTOMERS,PRODUCTS
+
+# Incremental mode (append only)
+python migrate.py --mode incremental
+
+# Dry run – connect to both DBs but write nothing
+python migrate.py --dry-run
+```
+
+## GitHub Actions
+
+### Daily schedule
+
+The workflow `.github/workflows/daily_migration.yml` runs automatically every day
+at **02:00 UTC** (`cron: "0 2 * * *"`).
+
+You can also trigger it manually from the **Actions** tab, optionally overriding
+the table list, mode, and dry-run flag.
+
+### Secrets required
+
+Add the following secrets to your repository (or the `production` environment):
+
+- `ORACLE_HOST`, `ORACLE_PORT`, `ORACLE_SERVICE`, `ORACLE_USER`, `ORACLE_PASSWORD`
+- `PG_HOST`, `PG_PORT`, `PG_DATABASE`, `PG_USER`, `PG_PASSWORD`
+
+### CI
+
+The workflow `.github/workflows/ci.yml` runs unit tests on every push and pull
+request.
+
+## Running tests
+
+```bash
+pytest tests/ -v
+```
+
+## How it works
+
+1. **Schema discovery** – the script queries `ALL_TAB_COLUMNS` in Oracle to
+   retrieve the full column list (name, data type, precision, scale, nullability).
+2. **Type mapping** – Oracle types are converted to the closest PostgreSQL
+   equivalents (see `type_mapping.py`).
+3. **DDL** – `CREATE TABLE IF NOT EXISTS` is executed in PostgreSQL.
+4. **Data transfer** – rows are streamed from Oracle in configurable batches and
+   bulk-inserted into PostgreSQL using `psycopg2.extras.execute_values`.
+5. **Full vs incremental** – in `full` mode the target table is truncated before
+   loading; in `incremental` mode rows are appended.

config.py

@@ -0,0 +1,117 @@
+"""Configuration helpers for the Oracle → PostgreSQL migration tool.
+
+All sensitive connection parameters are read from environment variables so
+that no credentials are stored in source code or configuration files.
+
+Required environment variables
+--------------------------------
+Oracle (source)
+    ORACLE_HOST       – hostname or IP of the Oracle server
+    ORACLE_PORT       – listener port (default: 1521)
+    ORACLE_SERVICE    – service name *or* SID
+    ORACLE_USER       – schema / user name
+    ORACLE_PASSWORD   – password
+
+PostgreSQL (destination)
+    PG_HOST           – hostname or IP of the PostgreSQL server
+    PG_PORT           – port (default: 5432)
+    PG_DATABASE       – database name
+    PG_SCHEMA         – target schema (default: public)
+    PG_USER           – user name
+    PG_PASSWORD       – password
+
+Optional
+    MIGRATION_TABLES  – comma-separated list of tables to migrate (default: all)
+    MIGRATION_BATCH_SIZE – rows fetched per batch (default: 10000)
+    MIGRATION_MODE    – 'full' (truncate+reload) or 'incremental' (default: full)
+    LOG_LEVEL         – Python logging level name (default: INFO)
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise EnvironmentError(
+            f"Required environment variable '{name}' is not set."
+        )
+    return value
+
+
+def _optional_env(name: str, default: str) -> str:
+    return os.environ.get(name, default)
+
+
+@dataclass(frozen=True)
+class OracleConfig:
+    host: str
+    port: int
+    service: str
+    user: str
+    password: str
+
+    @classmethod
+    def from_env(cls) -> "OracleConfig":
+        return cls(
+            host=_require_env("ORACLE_HOST"),
+            port=int(_optional_env("ORACLE_PORT", "1521")),
+            service=_require_env("ORACLE_SERVICE"),
+            user=_require_env("ORACLE_USER"),
+            password=_require_env("ORACLE_PASSWORD"),
+        )
+
+    @property
+    def dsn(self) -> str:
+        return f"{self.host}:{self.port}/{self.service}"
+
+
+@dataclass(frozen=True)
+class PostgresConfig:
+    host: str
+    port: int
+    database: str
+    user: str
+    password: str
+    schema: str = "public"
+
+    @classmethod
+    def from_env(cls) -> "PostgresConfig":
+        return cls(
+            host=_require_env("PG_HOST"),
+            port=int(_optional_env("PG_PORT", "5432")),
+            database=_require_env("PG_DATABASE"),
+            user=_require_env("PG_USER"),
+            password=_require_env("PG_PASSWORD"),
+            schema=_optional_env("PG_SCHEMA", "public"),
+        )
+
+
+@dataclass(frozen=True)
+class MigrationConfig:
+    oracle: OracleConfig
+    postgres: PostgresConfig
+    tables: list[str] = field(default_factory=list)
+    batch_size: int = 10_000
+    mode: str = "full"  # 'full' | 'incremental'
+    log_level: str = "INFO"
+
+    @classmethod
+    def from_env(cls) -> "MigrationConfig":
+        tables_raw = _optional_env("MIGRATION_TABLES", "")
+        tables = [t.strip() for t in tables_raw.split(",") if t.strip()]
+        batch_size = int(_optional_env("MIGRATION_BATCH_SIZE", "10000"))
+        mode = _optional_env("MIGRATION_MODE", "full").lower()
+        if mode not in ("full", "incremental"):
+            raise ValueError(f"MIGRATION_MODE must be 'full' or 'incremental', got '{mode}'")
+        return cls(
+            oracle=OracleConfig.from_env(),
+            postgres=PostgresConfig.from_env(),
+            tables=tables,
+            batch_size=batch_size,
+            mode=mode,
+            log_level=_optional_env("LOG_LEVEL", "INFO").upper(),
+        )